Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F1EB3200C06 for ; Fri, 27 Jan 2017 19:41:09 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id ED037160B5B; Fri, 27 Jan 2017 18:41:09 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E9A74160B40 for ; Fri, 27 Jan 2017 19:41:08 +0100 (CET) Received: (qmail 15811 invoked by uid 500); 27 Jan 2017 18:40:07 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 15061 invoked by uid 99); 27 Jan 2017 18:40:06 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 18:40:06 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 91702F4049; Fri, 27 Jan 2017 18:40:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Fri, 27 Jan 2017 18:40:23 -0000 Message-Id: <43aa5adf184440f88e8e8ae455666f3b@git.apache.org> In-Reply-To: <7fccdec439f94cbd8d0afe3fe9340207@git.apache.org> References: <7fccdec439f94cbd8d0afe3fe9340207@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [18/50] [abbrv] hadoop git commit: HDFS-11267. Avoid redefinition of storageDirs in NNStorage and cleanup its accessors in Storage. (Manoj Govindassamy via lei) archived-at: Fri, 27 Jan 2017 18:41:10 -0000 HDFS-11267. Avoid redefinition of storageDirs in NNStorage and cleanup its accessors in Storage. (Manoj Govindassamy via lei) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4f66655 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4f66655 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4f66655 Branch: refs/heads/YARN-3926 Commit: a4f66655ec22ca8c960f971f2b0cdafbd3430ad7 Parents: e9f1396 Author: Lei Xu Authored: Thu Dec 29 16:57:40 2016 +0800 Committer: Lei Xu Committed: Thu Dec 29 16:57:40 2016 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/server/common/Storage.java | 8 ++++++-- .../hdfs/server/datanode/BlockPoolSliceStorage.java | 6 +++--- .../hadoop/hdfs/server/datanode/DataStorage.java | 8 ++++---- .../apache/hadoop/hdfs/server/namenode/NNStorage.java | 14 ++++++-------- .../hadoop/hdfs/server/common/StorageAdapter.java | 2 +- .../server/datanode/TestBlockPoolSliceStorage.java | 2 +- 6 files changed, 21 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index c172289..f23a48a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -125,7 +125,7 @@ public abstract class Storage extends StorageInfo { } protected List storageDirs = - new CopyOnWriteArrayList(); + new CopyOnWriteArrayList<>(); private class DirIterator implements Iterator { final StorageDirType dirType; @@ -938,7 +938,11 @@ public abstract class Storage extends StorageInfo { public int getNumStorageDirs() { return storageDirs.size(); } - + + public List getStorageDirs() { + return storageDirs; + } + public StorageDirectory getStorageDir(int idx) { return storageDirs.get(idx); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index dd82a74..3203de2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -295,7 +295,7 @@ public class BlockPoolSliceStorage extends Storage { void remove(File absPathToRemove) { Preconditions.checkArgument(absPathToRemove.isAbsolute()); LOG.info("Removing block level storage: " + absPathToRemove); - for (Iterator it = this.storageDirs.iterator(); + for (Iterator it = getStorageDirs().iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); if (sd.getRoot().getAbsoluteFile().equals(absPathToRemove)) { @@ -788,7 +788,7 @@ public class BlockPoolSliceStorage extends Storage { */ public void clearTrash() { final List trashRoots = new ArrayList<>(); - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { File trashRoot = getTrashRootDir(sd); if (trashRoot.exists() && sd.getPreviousDir().exists()) { LOG.error("Trash and PreviousDir shouldn't both exist for storage " @@ -826,7 +826,7 @@ public class BlockPoolSliceStorage extends Storage { /** trash is enabled if at least one storage directory contains trash root */ @VisibleForTesting public boolean trashEnabled() { - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { if (getTrashRootDir(sd).exists()) { return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 12d9322..f410a85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -181,11 +181,11 @@ public class DataStorage extends Storage { } public void setRollingUpgradeMarker(String bpid) throws IOException { - getBPStorage(bpid).setRollingUpgradeMarkers(storageDirs); + getBPStorage(bpid).setRollingUpgradeMarkers(getStorageDirs()); } public void clearRollingUpgradeMarker(String bpid) throws IOException { - getBPStorage(bpid).clearRollingUpgradeMarkers(storageDirs); + getBPStorage(bpid).clearRollingUpgradeMarkers(getStorageDirs()); } /** @@ -493,7 +493,7 @@ public class DataStorage extends Storage { } StringBuilder errorMsgBuilder = new StringBuilder(); - for (Iterator it = this.storageDirs.iterator(); + for (Iterator it = getStorageDirs().iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); StorageLocation sdLocation = sd.getStorageLocation(); @@ -988,7 +988,7 @@ public class DataStorage extends Storage { // To handle finalizing a snapshot taken at datanode level while // upgrading to federation, if datanode level snapshot previous exists, // then finalize it. Else finalize the corresponding BP. - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { File prevDir = sd.getPreviousDir(); if (prevDir.exists()) { // data node level storage finalize http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index d6dd8ee..c79ba4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -172,8 +172,6 @@ public class NNStorage extends Storage implements Closeable, throws IOException { super(NodeType.NAME_NODE); - storageDirs = new CopyOnWriteArrayList<>(); - // this may modify the editsDirs, so copy before passing in setStorageDirectories(imageDirs, Lists.newArrayList(editsDirs), @@ -212,7 +210,7 @@ public class NNStorage extends Storage implements Closeable, @Override // Closeable public void close() throws IOException { unlockAll(); - storageDirs.clear(); + getStorageDirs().clear(); } /** @@ -295,7 +293,7 @@ public class NNStorage extends Storage implements Closeable, Collection fsEditsDirs, Collection sharedEditsDirs) throws IOException { - this.storageDirs.clear(); + getStorageDirs().clear(); this.removedStorageDirs.clear(); // Add all name dirs with appropriate NameNodeDirType @@ -873,7 +871,7 @@ public class NNStorage extends Storage implements Closeable, + sd.getRoot().getPath(), e); } - if (this.storageDirs.remove(sd)) { + if (getStorageDirs().remove(sd)) { this.removedStorageDirs.add(sd); } @@ -928,7 +926,7 @@ public class NNStorage extends Storage implements Closeable, // getCanonicalPath may need to call stat() or readlink() and it's likely // those calls would fail due to the same underlying IO problem. String absPath = f.getAbsolutePath(); - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { String dirPath = sd.getRoot().getAbsolutePath(); if (!dirPath.endsWith(File.separator)) { dirPath += File.separator; @@ -1140,14 +1138,14 @@ public class NNStorage extends Storage implements Closeable, @Override public void writeAll() throws IOException { this.layoutVersion = getServiceLayoutVersion(); - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { try { writeProperties(sd); } catch (Exception e) { LOG.warn("Error during write properties to the VERSION file to " + sd.toString(), e); reportErrorsOnDirectory(sd); - if (storageDirs.isEmpty()) { + if (getStorageDirs().isEmpty()) { throw new IOException("All the storage failed while writing " + "properties to VERSION file"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java index 3dd81b2..c2df9bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java @@ -33,7 +33,7 @@ public abstract class StorageAdapter { Storage s, int idx) { StorageDirectory dir = Mockito.spy(s.getStorageDir(idx)); - s.storageDirs.set(idx, dir); + s.getStorageDirs().set(idx, dir); return dir; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java index a8f8b6a..d9bbe88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java @@ -50,7 +50,7 @@ public class TestBlockPoolSliceStorage { String clusterId) { super(namespaceID, bpID, cTime, clusterId); addStorageDir(new StorageDirectory(new File("/tmp/dontcare/" + bpID))); - assertThat(storageDirs.size(), is(1)); + assertThat(getStorageDirs().size(), is(1)); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org