From commits-return-78760-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Tue Oct 2 16:53:56 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 5E6A018078F for ; Tue, 2 Oct 2018 16:53:55 +0200 (CEST) Received: (qmail 38894 invoked by uid 500); 2 Oct 2018 14:53:54 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 38461 invoked by uid 99); 2 Oct 2018 14:53:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Oct 2018 14:53:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1EE9AE11AA; Tue, 2 Oct 2018 14:53:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Tue, 02 Oct 2018 14:54:02 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [11/26] hbase-site git commit: Published site at 42aa3dd463c0d30a9b940d296b87316b5c67e1f5. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/37b8a04a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.SnapshotFiles.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.SnapshotFiles.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.SnapshotFiles.html index 9319626..201cbdc 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.SnapshotFiles.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.SnapshotFiles.html @@ -76,230 +76,226 @@ 068 069 private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class); 070 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); -071 private static long sequenceId = 0; -072 private static FileSystem fs; -073 private static Path rootDir; -074 -075 @BeforeClass -076 public static void startCluster() throws Exception { -077 UTIL.startMiniDFSCluster(1); -078 fs = UTIL.getDFSCluster().getFileSystem(); -079 rootDir = UTIL.getDefaultRootDirPath(); -080 } -081 -082 @AfterClass -083 public static void stopCluster() throws Exception { -084 UTIL.shutdownMiniDFSCluster(); -085 } -086 -087 @After -088 public void cleanupFiles() throws Exception { -089 // cleanup the snapshot directory -090 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); -091 fs.delete(snapshotDir, true); -092 } -093 -094 @Test -095 public void testLoadAndDelete() throws IOException { -096 // don't refresh the cache unless we tell it to -097 long period = Long.MAX_VALUE; -098 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, -099 "test-snapshot-file-cache-refresh", new SnapshotFiles()); -100 -101 createAndTestSnapshotV1(cache, "snapshot1a", false, true); -102 createAndTestSnapshotV1(cache, "snapshot1b", true, true); -103 -104 createAndTestSnapshotV2(cache, "snapshot2a", false, true); -105 createAndTestSnapshotV2(cache, "snapshot2b", true, true); -106 } -107 -108 @Test -109 public void testReloadModifiedDirectory() throws IOException { -110 // don't refresh the cache unless we tell it to -111 long period = Long.MAX_VALUE; -112 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, -113 "test-snapshot-file-cache-refresh", new SnapshotFiles()); -114 -115 createAndTestSnapshotV1(cache, "snapshot1", false, true); -116 // now delete the snapshot and add a file with a different name -117 createAndTestSnapshotV1(cache, "snapshot1", false, false); -118 -119 createAndTestSnapshotV2(cache, "snapshot2", false, true); -120 // now delete the snapshot and add a file with a different name -121 createAndTestSnapshotV2(cache, "snapshot2", false, false); -122 } -123 -124 @Test -125 public void testSnapshotTempDirReload() throws IOException { -126 long period = Long.MAX_VALUE; -127 // This doesn't refresh cache until we invoke it explicitly -128 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, -129 "test-snapshot-file-cache-refresh", new SnapshotFiles()); -130 -131 // Add a new non-tmp snapshot -132 createAndTestSnapshotV1(cache, "snapshot0v1", false, false); -133 createAndTestSnapshotV1(cache, "snapshot0v2", false, false); -134 -135 // Add a new tmp snapshot -136 createAndTestSnapshotV2(cache, "snapshot1", true, false); -137 -138 // Add another tmp snapshot -139 createAndTestSnapshotV2(cache, "snapshot2", true, false); -140 } -141 -142 @Test -143 public void testWeNeverCacheTmpDirAndLoadIt() throws Exception { -144 -145 final AtomicInteger count = new AtomicInteger(0); -146 // don't refresh the cache unless we tell it to -147 long period = Long.MAX_VALUE; -148 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, -149 "test-snapshot-file-cache-refresh", new SnapshotFiles()) { -150 @Override -151 List<String> getSnapshotsInProgress(final SnapshotManager snapshotManager) -152 throws IOException { -153 List<String> result = super.getSnapshotsInProgress(snapshotManager); -154 count.incrementAndGet(); -155 return result; -156 } -157 -158 @Override public void triggerCacheRefreshForTesting() { -159 super.triggerCacheRefreshForTesting(); -160 } -161 }; -162 -163 SnapshotMock.SnapshotBuilder complete = -164 createAndTestSnapshotV1(cache, "snapshot", false, false); -165 -166 SnapshotMock.SnapshotBuilder inProgress = -167 createAndTestSnapshotV1(cache, "snapshotInProgress", true, false); +071 private static FileSystem fs; +072 private static Path rootDir; +073 +074 @BeforeClass +075 public static void startCluster() throws Exception { +076 UTIL.startMiniDFSCluster(1); +077 fs = UTIL.getDFSCluster().getFileSystem(); +078 rootDir = UTIL.getDefaultRootDirPath(); +079 } +080 +081 @AfterClass +082 public static void stopCluster() throws Exception { +083 UTIL.shutdownMiniDFSCluster(); +084 } +085 +086 @After +087 public void cleanupFiles() throws Exception { +088 // cleanup the snapshot directory +089 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); +090 fs.delete(snapshotDir, true); +091 } +092 +093 @Test +094 public void testLoadAndDelete() throws IOException { +095 // don't refresh the cache unless we tell it to +096 long period = Long.MAX_VALUE; +097 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, +098 "test-snapshot-file-cache-refresh", new SnapshotFiles()); +099 +100 createAndTestSnapshotV1(cache, "snapshot1a", false, true); +101 createAndTestSnapshotV1(cache, "snapshot1b", true, true); +102 +103 createAndTestSnapshotV2(cache, "snapshot2a", false, true); +104 createAndTestSnapshotV2(cache, "snapshot2b", true, true); +105 } +106 +107 @Test +108 public void testReloadModifiedDirectory() throws IOException { +109 // don't refresh the cache unless we tell it to +110 long period = Long.MAX_VALUE; +111 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, +112 "test-snapshot-file-cache-refresh", new SnapshotFiles()); +113 +114 createAndTestSnapshotV1(cache, "snapshot1", false, true); +115 // now delete the snapshot and add a file with a different name +116 createAndTestSnapshotV1(cache, "snapshot1", false, false); +117 +118 createAndTestSnapshotV2(cache, "snapshot2", false, true); +119 // now delete the snapshot and add a file with a different name +120 createAndTestSnapshotV2(cache, "snapshot2", false, false); +121 } +122 +123 @Test +124 public void testSnapshotTempDirReload() throws IOException { +125 long period = Long.MAX_VALUE; +126 // This doesn't refresh cache until we invoke it explicitly +127 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, +128 "test-snapshot-file-cache-refresh", new SnapshotFiles()); +129 +130 // Add a new non-tmp snapshot +131 createAndTestSnapshotV1(cache, "snapshot0v1", false, false); +132 createAndTestSnapshotV1(cache, "snapshot0v2", false, false); +133 +134 // Add a new tmp snapshot +135 createAndTestSnapshotV2(cache, "snapshot1", true, false); +136 +137 // Add another tmp snapshot +138 createAndTestSnapshotV2(cache, "snapshot2", true, false); +139 } +140 +141 @Test +142 public void testWeNeverCacheTmpDirAndLoadIt() throws Exception { +143 +144 final AtomicInteger count = new AtomicInteger(0); +145 // don't refresh the cache unless we tell it to +146 long period = Long.MAX_VALUE; +147 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, +148 "test-snapshot-file-cache-refresh", new SnapshotFiles()) { +149 @Override +150 List<String> getSnapshotsInProgress(final SnapshotManager snapshotManager) +151 throws IOException { +152 List<String> result = super.getSnapshotsInProgress(snapshotManager); +153 count.incrementAndGet(); +154 return result; +155 } +156 +157 @Override public void triggerCacheRefreshForTesting() { +158 super.triggerCacheRefreshForTesting(); +159 } +160 }; +161 +162 SnapshotMock.SnapshotBuilder complete = +163 createAndTestSnapshotV1(cache, "snapshot", false, false); +164 +165 int countBeforeCheck = count.get(); +166 +167 FSUtils.logFileSystemState(fs, rootDir, LOG); 168 -169 int countBeforeCheck = count.get(); -170 -171 FSUtils.logFileSystemState(fs, rootDir, LOG); -172 -173 List<FileStatus> allStoreFiles = getStoreFilesForSnapshot(complete); -174 Iterable<FileStatus> deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); -175 assertTrue(Iterables.isEmpty(deletableFiles)); -176 // no need for tmp dir check as all files are accounted for. -177 assertEquals(0, count.get() - countBeforeCheck); -178 -179 -180 // add a random file to make sure we refresh -181 FileStatus randomFile = mockStoreFile(UTIL.getRandomUUID().toString()); -182 allStoreFiles.add(randomFile); -183 deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); -184 assertEquals(randomFile, Iterables.getOnlyElement(deletableFiles)); -185 assertEquals(1, count.get() - countBeforeCheck); // we check the tmp directory -186 } -187 -188 private List<FileStatus> getStoreFilesForSnapshot(SnapshotMock.SnapshotBuilder builder) -189 throws IOException { -190 final List<FileStatus> allStoreFiles = Lists.newArrayList(); -191 SnapshotReferenceUtil -192 .visitReferencedFiles(UTIL.getConfiguration(), fs, builder.getSnapshotsDir(), -193 new SnapshotReferenceUtil.SnapshotVisitor() { -194 @Override public void storeFile(RegionInfo regionInfo, String familyName, -195 SnapshotProtos.SnapshotRegionManifest.StoreFile storeFile) throws IOException { -196 FileStatus status = mockStoreFile(storeFile.getName()); -197 allStoreFiles.add(status); -198 } -199 }); -200 return allStoreFiles; -201 } -202 -203 private FileStatus mockStoreFile(String storeFileName) { -204 FileStatus status = mock(FileStatus.class); -205 Path path = mock(Path.class); -206 when(path.getName()).thenReturn(storeFileName); -207 when(status.getPath()).thenReturn(path); -208 return status; -209 } -210 -211 class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { -212 @Override -213 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { -214 Collection<String> files = new HashSet<>(); -215 files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); -216 return files; -217 } -218 } -219 -220 private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, -221 final String name, final boolean tmp, final boolean removeOnExit) throws IOException { -222 SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); -223 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); -224 createAndTestSnapshot(cache, builder, tmp, removeOnExit); -225 return builder; -226 } -227 -228 private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, -229 final boolean tmp, final boolean removeOnExit) throws IOException { -230 SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); -231 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); -232 createAndTestSnapshot(cache, builder, tmp, removeOnExit); -233 } -234 -235 private void createAndTestSnapshot(final SnapshotFileCache cache, -236 final SnapshotMock.SnapshotBuilder builder, -237 final boolean tmp, final boolean removeOnExit) throws IOException { -238 List<Path> files = new ArrayList<>(); -239 for (int i = 0; i < 3; ++i) { -240 for (Path filePath: builder.addRegion()) { -241 String fileName = filePath.getName(); -242 if (tmp) { -243 // We should be able to find all the files while the snapshot creation is in-progress -244 FSUtils.logFileSystemState(fs, rootDir, LOG); -245 Iterable<FileStatus> nonSnapshot = getNonSnapshotFiles(cache, filePath); -246 assertFalse("Cache didn't find " + fileName, Iterables.contains(nonSnapshot, fileName)); -247 } -248 files.add(filePath); -249 } +169 List<FileStatus> allStoreFiles = getStoreFilesForSnapshot(complete); +170 Iterable<FileStatus> deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); +171 assertTrue(Iterables.isEmpty(deletableFiles)); +172 // no need for tmp dir check as all files are accounted for. +173 assertEquals(0, count.get() - countBeforeCheck); +174 +175 +176 // add a random file to make sure we refresh +177 FileStatus randomFile = mockStoreFile(UTIL.getRandomUUID().toString()); +178 allStoreFiles.add(randomFile); +179 deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); +180 assertEquals(randomFile, Iterables.getOnlyElement(deletableFiles)); +181 assertEquals(1, count.get() - countBeforeCheck); // we check the tmp directory +182 } +183 +184 private List<FileStatus> getStoreFilesForSnapshot(SnapshotMock.SnapshotBuilder builder) +185 throws IOException { +186 final List<FileStatus> allStoreFiles = Lists.newArrayList(); +187 SnapshotReferenceUtil +188 .visitReferencedFiles(UTIL.getConfiguration(), fs, builder.getSnapshotsDir(), +189 new SnapshotReferenceUtil.SnapshotVisitor() { +190 @Override public void storeFile(RegionInfo regionInfo, String familyName, +191 SnapshotProtos.SnapshotRegionManifest.StoreFile storeFile) throws IOException { +192 FileStatus status = mockStoreFile(storeFile.getName()); +193 allStoreFiles.add(status); +194 } +195 }); +196 return allStoreFiles; +197 } +198 +199 private FileStatus mockStoreFile(String storeFileName) { +200 FileStatus status = mock(FileStatus.class); +201 Path path = mock(Path.class); +202 when(path.getName()).thenReturn(storeFileName); +203 when(status.getPath()).thenReturn(path); +204 return status; +205 } +206 +207 class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { +208 @Override +209 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { +210 Collection<String> files = new HashSet<>(); +211 files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); +212 return files; +213 } +214 } +215 +216 private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, +217 final String name, final boolean tmp, final boolean removeOnExit) throws IOException { +218 SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); +219 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); +220 createAndTestSnapshot(cache, builder, tmp, removeOnExit); +221 return builder; +222 } +223 +224 private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, +225 final boolean tmp, final boolean removeOnExit) throws IOException { +226 SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); +227 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); +228 createAndTestSnapshot(cache, builder, tmp, removeOnExit); +229 } +230 +231 private void createAndTestSnapshot(final SnapshotFileCache cache, +232 final SnapshotMock.SnapshotBuilder builder, +233 final boolean tmp, final boolean removeOnExit) throws IOException { +234 List<Path> files = new ArrayList<>(); +235 for (int i = 0; i < 3; ++i) { +236 for (Path filePath: builder.addRegion()) { +237 if (tmp) { +238 // We should be able to find all the files while the snapshot creation is in-progress +239 FSUtils.logFileSystemState(fs, rootDir, LOG); +240 assertFalse("Cache didn't find " + filePath, +241 contains(getNonSnapshotFiles(cache, filePath), filePath)); +242 } +243 files.add(filePath); +244 } +245 } +246 +247 // Finalize the snapshot +248 if (!tmp) { +249 builder.commit(); 250 } 251 -252 // Finalize the snapshot -253 if (!tmp) { -254 builder.commit(); +252 // Make sure that all files are still present +253 for (Path path: files) { +254 assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path)); 255 } 256 -257 // Make sure that all files are still present -258 for (Path path: files) { -259 Iterable<FileStatus> nonSnapshotFiles = getNonSnapshotFiles(cache, path); -260 assertFalse("Cache didn't find " + path.getName(), -261 Iterables.contains(nonSnapshotFiles, path.getName())); -262 } -263 -264 FSUtils.logFileSystemState(fs, rootDir, LOG); -265 if (removeOnExit) { -266 LOG.debug("Deleting snapshot."); -267 fs.delete(builder.getSnapshotsDir(), true); -268 FSUtils.logFileSystemState(fs, rootDir, LOG); +257 FSUtils.logFileSystemState(fs, rootDir, LOG); +258 if (removeOnExit) { +259 LOG.debug("Deleting snapshot."); +260 fs.delete(builder.getSnapshotsDir(), true); +261 FSUtils.logFileSystemState(fs, rootDir, LOG); +262 +263 // then trigger a refresh +264 cache.triggerCacheRefreshForTesting(); +265 // and not it shouldn't find those files +266 for (Path filePath: files) { +267 assertTrue("Cache found '" + filePath + "', but it shouldn't have.", +268 contains(getNonSnapshotFiles(cache, filePath), filePath)); 269 -270 // The files should be in cache until next refresh -271 for (Path filePath: files) { -272 Iterable<FileStatus> nonSnapshotFiles = getNonSnapshotFiles(cache, filePath); -273 assertFalse("Cache didn't find " + filePath.getName(), Iterables.contains(nonSnapshotFiles, -274 filePath.getName())); -275 } -276 -277 // then trigger a refresh -278 cache.triggerCacheRefreshForTesting(); -279 // and not it shouldn't find those files -280 for (Path filePath: files) { -281 Iterable<FileStatus> nonSnapshotFiles = getNonSnapshotFiles(cache, filePath); -282 assertTrue("Cache found '" + filePath.getName() + "', but it shouldn't have.", -283 !Iterables.contains(nonSnapshotFiles, filePath.getName())); -284 } -285 } -286 } -287 -288 private Iterable<FileStatus> getNonSnapshotFiles(SnapshotFileCache cache, Path storeFile) -289 throws IOException { -290 return cache.getUnreferencedFiles( -291 Arrays.asList(FSUtils.listStatus(fs, storeFile.getParent())), null -292 ); -293 } -294} +270 } +271 } +272 } +273 +274 private static boolean contains(Iterable<FileStatus> files, Path filePath) { +275 for (FileStatus status: files) { +276 LOG.debug("debug in contains, 3.1: " + status.getPath() + " filePath:" + filePath); +277 if (filePath.equals(status.getPath())) { +278 return true; +279 } +280 } +281 return false; +282 } +283 +284 private static Iterable<FileStatus> getNonSnapshotFiles(SnapshotFileCache cache, Path storeFile) +285 throws IOException { +286 return cache.getUnreferencedFiles( +287 Arrays.asList(FSUtils.listStatus(fs, storeFile.getParent())), null +288 ); +289 } +290}