From commits-return-69714-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Sat Mar 17 15:53:47 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id DCE741807A6 for ; Sat, 17 Mar 2018 15:53:42 +0100 (CET) Received: (qmail 98860 invoked by uid 500); 17 Mar 2018 14:53:40 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 97700 invoked by uid 99); 17 Mar 2018 14:53:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 Mar 2018 14:53:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AD8ABF66B9; Sat, 17 Mar 2018 14:53:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 17 Mar 2018 14:54:06 -0000 Message-Id: <52aaca6d265548479b318347e60b1a38@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [30/51] [partial] hbase-site git commit: Published site at 00095a2ef9442e3fd86c04876c9d91f2f8b23ad8. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/bd675fa3/devapidocs/src-html/org/apache/hadoop/hbase/backup/HFileArchiver.FileStatusConverter.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/backup/HFileArchiver.FileStatusConverter.html b/devapidocs/src-html/org/apache/hadoop/hbase/backup/HFileArchiver.FileStatusConverter.html index f47d627..c3d225c 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/backup/HFileArchiver.FileStatusConverter.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/backup/HFileArchiver.FileStatusConverter.html @@ -117,219 +117,219 @@ 109 */ 110 public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir) 111 throws IOException { -112 if (LOG.isDebugEnabled()) { -113 LOG.debug("ARCHIVING " + regionDir.toString()); -114 } -115 -116 // otherwise, we archive the files -117 // make sure we can archive -118 if (tableDir == null || regionDir == null) { -119 LOG.error("No archive directory could be found because tabledir (" + tableDir -120 + ") or regiondir (" + regionDir + "was null. Deleting files instead."); -121 deleteRegionWithoutArchiving(fs, regionDir); -122 // we should have archived, but failed to. Doesn't matter if we deleted -123 // the archived files correctly or not. -124 return false; -125 } -126 -127 // make sure the regiondir lives under the tabledir -128 Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString())); -129 Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, -130 FSUtils.getTableName(tableDir), -131 regionDir.getName()); -132 -133 FileStatusConverter getAsFile = new FileStatusConverter(fs); -134 // otherwise, we attempt to archive the store files -135 -136 // build collection of just the store directories to archive -137 Collection<File> toArchive = new ArrayList<>(); -138 final PathFilter dirFilter = new FSUtils.DirFilter(fs); -139 PathFilter nonHidden = new PathFilter() { -140 @Override -141 public boolean accept(Path file) { -142 return dirFilter.accept(file) && !file.getName().toString().startsWith("."); -143 } -144 }; -145 FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden); -146 // if there no files, we can just delete the directory and return; -147 if (storeDirs == null) { -148 LOG.debug("Region directory " + regionDir + " empty."); -149 return deleteRegionWithoutArchiving(fs, regionDir); -150 } -151 -152 // convert the files in the region to a File -153 toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile)); -154 LOG.debug("Archiving " + toArchive); -155 List<File> failedArchive = resolveAndArchive(fs, regionArchiveDir, toArchive, -156 EnvironmentEdgeManager.currentTime()); -157 if (!failedArchive.isEmpty()) { -158 throw new FailedArchiveException("Failed to archive/delete all the files for region:" -159 + regionDir.getName() + " into " + regionArchiveDir -160 + ". Something is probably awry on the filesystem.", -161 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); -162 } -163 // if that was successful, then we delete the region -164 return deleteRegionWithoutArchiving(fs, regionDir); -165 } -166 -167 /** -168 * Remove from the specified region the store files of the specified column family, -169 * either by archiving them or outright deletion -170 * @param fs the filesystem where the store files live -171 * @param conf {@link Configuration} to examine to determine the archive directory -172 * @param parent Parent region hosting the store files -173 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) -174 * @param family the family hosting the store files -175 * @throws IOException if the files could not be correctly disposed. -176 */ -177 public static void archiveFamily(FileSystem fs, Configuration conf, -178 RegionInfo parent, Path tableDir, byte[] family) throws IOException { -179 Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); -180 archiveFamilyByFamilyDir(fs, conf, parent, familyDir, family); -181 } -182 -183 /** -184 * Removes from the specified region the store files of the specified column family, -185 * either by archiving them or outright deletion -186 * @param fs the filesystem where the store files live -187 * @param conf {@link Configuration} to examine to determine the archive directory -188 * @param parent Parent region hosting the store files -189 * @param familyDir {@link Path} to where the family is being stored -190 * @param family the family hosting the store files -191 * @throws IOException if the files could not be correctly disposed. -192 */ -193 public static void archiveFamilyByFamilyDir(FileSystem fs, Configuration conf, -194 RegionInfo parent, Path familyDir, byte[] family) throws IOException { -195 FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); -196 if (storeFiles == null) { -197 LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + -198 ", family=" + Bytes.toString(family)); -199 return; -200 } -201 -202 FileStatusConverter getAsFile = new FileStatusConverter(fs); -203 Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); -204 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family); -205 -206 // do the actual archive -207 List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, toArchive, -208 EnvironmentEdgeManager.currentTime()); -209 if (!failedArchive.isEmpty()){ -210 throw new FailedArchiveException("Failed to archive/delete all the files for region:" -211 + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) -212 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", -213 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); -214 } -215 } -216 -217 /** -218 * Remove the store files, either by archiving them or outright deletion -219 * @param conf {@link Configuration} to examine to determine the archive directory -220 * @param fs the filesystem where the store files live -221 * @param regionInfo {@link RegionInfo} of the region hosting the store files -222 * @param family the family hosting the store files -223 * @param compactedFiles files to be disposed of. No further reading of these files should be -224 * attempted; otherwise likely to cause an {@link IOException} -225 * @throws IOException if the files could not be correctly disposed. -226 */ -227 public static void archiveStoreFiles(Configuration conf, FileSystem fs, RegionInfo regionInfo, -228 Path tableDir, byte[] family, Collection<HStoreFile> compactedFiles) -229 throws IOException, FailedArchiveException { -230 -231 // sometimes in testing, we don't have rss, so we need to check for that -232 if (fs == null) { -233 LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:" -234 + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family)); -235 deleteStoreFilesWithoutArchiving(compactedFiles); -236 return; -237 } -238 -239 // short circuit if we don't have any files to delete -240 if (compactedFiles.isEmpty()) { -241 LOG.debug("No store files to dispose, done!"); -242 return; -243 } -244 -245 // build the archive path -246 if (regionInfo == null || family == null) throw new IOException( -247 "Need to have a region and a family to archive from."); +112 LOG.debug("ARCHIVING {}", rootdir.toString()); +113 +114 // otherwise, we archive the files +115 // make sure we can archive +116 if (tableDir == null || regionDir == null) { +117 LOG.error("No archive directory could be found because tabledir (" + tableDir +118 + ") or regiondir (" + regionDir + "was null. Deleting files instead."); +119 deleteRegionWithoutArchiving(fs, regionDir); +120 // we should have archived, but failed to. Doesn't matter if we deleted +121 // the archived files correctly or not. +122 return false; +123 } +124 +125 // make sure the regiondir lives under the tabledir +126 Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString())); +127 Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, +128 FSUtils.getTableName(tableDir), +129 regionDir.getName()); +130 +131 FileStatusConverter getAsFile = new FileStatusConverter(fs); +132 // otherwise, we attempt to archive the store files +133 +134 // build collection of just the store directories to archive +135 Collection<File> toArchive = new ArrayList<>(); +136 final PathFilter dirFilter = new FSUtils.DirFilter(fs); +137 PathFilter nonHidden = new PathFilter() { +138 @Override +139 public boolean accept(Path file) { +140 return dirFilter.accept(file) && !file.getName().toString().startsWith("."); +141 } +142 }; +143 FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden); +144 // if there no files, we can just delete the directory and return; +145 if (storeDirs == null) { +146 LOG.debug("Directory {} empty.", regionDir); +147 return deleteRegionWithoutArchiving(fs, regionDir); +148 } +149 +150 // convert the files in the region to a File +151 toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile)); +152 LOG.debug("Archiving " + toArchive); +153 List<File> failedArchive = resolveAndArchive(fs, regionArchiveDir, toArchive, +154 EnvironmentEdgeManager.currentTime()); +155 if (!failedArchive.isEmpty()) { +156 throw new FailedArchiveException("Failed to archive/delete all the files for region:" +157 + regionDir.getName() + " into " + regionArchiveDir +158 + ". Something is probably awry on the filesystem.", +159 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); +160 } +161 // if that was successful, then we delete the region +162 return deleteRegionWithoutArchiving(fs, regionDir); +163 } +164 +165 /** +166 * Remove from the specified region the store files of the specified column family, +167 * either by archiving them or outright deletion +168 * @param fs the filesystem where the store files live +169 * @param conf {@link Configuration} to examine to determine the archive directory +170 * @param parent Parent region hosting the store files +171 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) +172 * @param family the family hosting the store files +173 * @throws IOException if the files could not be correctly disposed. +174 */ +175 public static void archiveFamily(FileSystem fs, Configuration conf, +176 RegionInfo parent, Path tableDir, byte[] family) throws IOException { +177 Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); +178 archiveFamilyByFamilyDir(fs, conf, parent, familyDir, family); +179 } +180 +181 /** +182 * Removes from the specified region the store files of the specified column family, +183 * either by archiving them or outright deletion +184 * @param fs the filesystem where the store files live +185 * @param conf {@link Configuration} to examine to determine the archive directory +186 * @param parent Parent region hosting the store files +187 * @param familyDir {@link Path} to where the family is being stored +188 * @param family the family hosting the store files +189 * @throws IOException if the files could not be correctly disposed. +190 */ +191 public static void archiveFamilyByFamilyDir(FileSystem fs, Configuration conf, +192 RegionInfo parent, Path familyDir, byte[] family) throws IOException { +193 FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); +194 if (storeFiles == null) { +195 LOG.debug("No files to dispose of in {}, family={}", parent.getRegionNameAsString(), +196 Bytes.toString(family)); +197 return; +198 } +199 +200 FileStatusConverter getAsFile = new FileStatusConverter(fs); +201 Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); +202 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family); +203 +204 // do the actual archive +205 List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, toArchive, +206 EnvironmentEdgeManager.currentTime()); +207 if (!failedArchive.isEmpty()){ +208 throw new FailedArchiveException("Failed to archive/delete all the files for region:" +209 + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) +210 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", +211 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); +212 } +213 } +214 +215 /** +216 * Remove the store files, either by archiving them or outright deletion +217 * @param conf {@link Configuration} to examine to determine the archive directory +218 * @param fs the filesystem where the store files live +219 * @param regionInfo {@link RegionInfo} of the region hosting the store files +220 * @param family the family hosting the store files +221 * @param compactedFiles files to be disposed of. No further reading of these files should be +222 * attempted; otherwise likely to cause an {@link IOException} +223 * @throws IOException if the files could not be correctly disposed. +224 */ +225 public static void archiveStoreFiles(Configuration conf, FileSystem fs, RegionInfo regionInfo, +226 Path tableDir, byte[] family, Collection<HStoreFile> compactedFiles) +227 throws IOException, FailedArchiveException { +228 +229 // sometimes in testing, we don't have rss, so we need to check for that +230 if (fs == null) { +231 LOG.warn("Passed filesystem is null, so just deleting files without archiving for {}," + +232 "family={}", Bytes.toString(regionInfo.getRegionName()), Bytes.toString(family)); +233 deleteStoreFilesWithoutArchiving(compactedFiles); +234 return; +235 } +236 +237 // short circuit if we don't have any files to delete +238 if (compactedFiles.isEmpty()) { +239 LOG.debug("No files to dispose of, done!"); +240 return; +241 } +242 +243 // build the archive path +244 if (regionInfo == null || family == null) throw new IOException( +245 "Need to have a region and a family to archive from."); +246 +247 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); 248 -249 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); -250 -251 // make sure we don't archive if we can't and that the archive dir exists -252 if (!fs.mkdirs(storeArchiveDir)) { -253 throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" -254 + Bytes.toString(family) + ", deleting compacted files instead."); -255 } -256 -257 // otherwise we attempt to archive the store files -258 LOG.debug("Archiving compacted store files."); -259 -260 // Wrap the storefile into a File -261 StoreToFile getStorePath = new StoreToFile(fs); -262 Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath); -263 -264 // do the actual archive -265 List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, storeFiles, -266 EnvironmentEdgeManager.currentTime()); -267 -268 if (!failedArchive.isEmpty()){ -269 throw new FailedArchiveException("Failed to archive/delete all the files for region:" -270 + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family) -271 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", -272 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); -273 } -274 } -275 -276 /** -277 * Archive the store file -278 * @param fs the filesystem where the store files live -279 * @param regionInfo region hosting the store files -280 * @param conf {@link Configuration} to examine to determine the archive directory -281 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) -282 * @param family the family hosting the store files -283 * @param storeFile file to be archived -284 * @throws IOException if the files could not be correctly disposed. -285 */ -286 public static void archiveStoreFile(Configuration conf, FileSystem fs, RegionInfo regionInfo, -287 Path tableDir, byte[] family, Path storeFile) throws IOException { -288 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); -289 // make sure we don't archive if we can't and that the archive dir exists -290 if (!fs.mkdirs(storeArchiveDir)) { -291 throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" -292 + Bytes.toString(family) + ", deleting compacted files instead."); -293 } -294 -295 // do the actual archive -296 long start = EnvironmentEdgeManager.currentTime(); -297 File file = new FileablePath(fs, storeFile); -298 if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { -299 throw new IOException("Failed to archive/delete the file for region:" -300 + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) -301 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); -302 } -303 } -304 -305 /** -306 * Resolve any conflict with an existing archive file via timestamp-append -307 * renaming of the existing file and then archive the passed in files. -308 * @param fs {@link FileSystem} on which to archive the files -309 * @param baseArchiveDir base archive directory to store the files. If any of -310 * the files to archive are directories, will append the name of the -311 * directory to the base archive directory name, creating a parallel -312 * structure. -313 * @param toArchive files/directories that need to be archvied -314 * @param start time the archiving started - used for resolving archive -315 * conflicts. -316 * @return the list of failed to archive files. -317 * @throws IOException if an unexpected file operation exception occurred -318 */ -319 private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir, -320 Collection<File> toArchive, long start) throws IOException { -321 // short circuit if no files to move -322 if (toArchive.isEmpty()) return Collections.emptyList(); +249 // make sure we don't archive if we can't and that the archive dir exists +250 if (!fs.mkdirs(storeArchiveDir)) { +251 throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" +252 + Bytes.toString(family) + ", deleting compacted files instead."); +253 } +254 +255 // otherwise we attempt to archive the store files +256 LOG.debug("Archiving compacted files."); +257 +258 // Wrap the storefile into a File +259 StoreToFile getStorePath = new StoreToFile(fs); +260 Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath); +261 +262 // do the actual archive +263 List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, storeFiles, +264 EnvironmentEdgeManager.currentTime()); +265 +266 if (!failedArchive.isEmpty()){ +267 throw new FailedArchiveException("Failed to archive/delete all the files for region:" +268 + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family) +269 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", +270 Collections2.transform(failedArchive, FUNC_FILE_TO_PATH)); +271 } +272 } +273 +274 /** +275 * Archive the store file +276 * @param fs the filesystem where the store files live +277 * @param regionInfo region hosting the store files +278 * @param conf {@link Configuration} to examine to determine the archive directory +279 * @param tableDir {@link Path} to where the table is being stored (for building the archive path) +280 * @param family the family hosting the store files +281 * @param storeFile file to be archived +282 * @throws IOException if the files could not be correctly disposed. +283 */ +284 public static void archiveStoreFile(Configuration conf, FileSystem fs, RegionInfo regionInfo, +285 Path tableDir, byte[] family, Path storeFile) throws IOException { +286 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); +287 // make sure we don't archive if we can't and that the archive dir exists +288 if (!fs.mkdirs(storeArchiveDir)) { +289 throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" +290 + Bytes.toString(family) + ", deleting compacted files instead."); +291 } +292 +293 // do the actual archive +294 long start = EnvironmentEdgeManager.currentTime(); +295 File file = new FileablePath(fs, storeFile); +296 if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { +297 throw new IOException("Failed to archive/delete the file for region:" +298 + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) +299 + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); +300 } +301 } +302 +303 /** +304 * Resolve any conflict with an existing archive file via timestamp-append +305 * renaming of the existing file and then archive the passed in files. +306 * @param fs {@link FileSystem} on which to archive the files +307 * @param baseArchiveDir base archive directory to store the files. If any of +308 * the files to archive are directories, will append the name of the +309 * directory to the base archive directory name, creating a parallel +310 * structure. +311 * @param toArchive files/directories that need to be archvied +312 * @param start time the archiving started - used for resolving archive +313 * conflicts. +314 * @return the list of failed to archive files. +315 * @throws IOException if an unexpected file operation exception occurred +316 */ +317 private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir, +318 Collection<File> toArchive, long start) throws IOException { +319 // short circuit if no files to move +320 if (toArchive.isEmpty()) { +321 return Collections.emptyList(); +322 } 323 -324 if (LOG.isTraceEnabled()) LOG.trace("moving files to the archive directory: " + baseArchiveDir); +324 LOG.trace("Moving files to the archive directory {}", baseArchiveDir); 325 326 // make sure the archive directory exists 327 if (!fs.exists(baseArchiveDir)) { @@ -337,7 +337,7 @@ 329 throw new IOException("Failed to create the archive directory:" + baseArchiveDir 330 + ", quitting archive attempt."); 331 } -332 if (LOG.isTraceEnabled()) LOG.trace("Created archive directory:" + baseArchiveDir); +332 LOG.trace("Created archive directory {}", baseArchiveDir); 333 } 334 335 List<File> failures = new ArrayList<>(); @@ -345,7 +345,7 @@ 337 for (File file : toArchive) { 338 // if its a file archive it 339 try { -340 if (LOG.isTraceEnabled()) LOG.trace("Archiving: " + file); +340 LOG.trace("Archiving {}", file); 341 if (file.isFile()) { 342 // attempt to archive the file 343 if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) { @@ -354,7 +354,7 @@ 346 } 347 } else { 348 // otherwise its a directory and we need to archive all files -349 if (LOG.isTraceEnabled()) LOG.trace(file + " is a directory, archiving children files"); +349 LOG.trace("{} is a directory, archiving children files", file); 350 // so we add the directory name to the one base archive 351 Path parentArchiveDir = new Path(baseArchiveDir, file.getName()); 352 // and then get all the files from that directory and attempt to @@ -363,7 +363,7 @@ 355 failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start)); 356 } 357 } catch (IOException e) { -358 LOG.warn("Failed to archive " + file, e); +358 LOG.warn("Failed to archive {}", file, e); 359 failures.add(file); 360 } 361 } @@ -393,319 +393,312 @@ 385 // really, really unlikely situtation, where we get the same name for the existing file, but 386 // is included just for that 1 in trillion chance. 387 if (fs.exists(archiveFile)) { -388 if (LOG.isDebugEnabled()) { -389 LOG.debug("File:" + archiveFile + " already exists in archive, moving to " -390 + "timestamped backup and overwriting current."); -391 } -392 -393 // move the archive file to the stamped backup -394 Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime); -395 if (!fs.rename(archiveFile, backedupArchiveFile)) { -396 LOG.error("Could not rename archive file to backup: " + backedupArchiveFile -397 + ", deleting existing file in favor of newer."); -398 // try to delete the exisiting file, if we can't rename it -399 if (!fs.delete(archiveFile, false)) { -400 throw new IOException("Couldn't delete existing archive file (" + archiveFile -401 + ") or rename it to the backup file (" + backedupArchiveFile -402 + ") to make room for similarly named file."); -403 } -404 } -405 LOG.debug("Backed up archive file from " + archiveFile); -406 } +388 LOG.debug("{} already exists in archive, moving to timestamped backup and " + +389 "overwriting current.", archiveFile); +390 +391 // move the archive file to the stamped backup +392 Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime); +393 if (!fs.rename(archiveFile, backedupArchiveFile)) { +394 LOG.error("Could not rename archive file to backup: " + backedupArchiveFile +395 + ", deleting existing file in favor of newer."); +396 // try to delete the exisiting file, if we can't rename it +397 if (!fs.delete(archiveFile, false)) { +398 throw new IOException("Couldn't delete existing archive file (" + archiveFile +399 + ") or rename it to the backup file (" + backedupArchiveFile +400 + ") to make room for similarly named file."); +401 } +402 } +403 LOG.debug("Backed up archive file from " + archiveFile); +404 } +405 +406 LOG.trace("No existing file in archive for {}, free to archive original file.", archiveFile); 407 -408 if (LOG.isTraceEnabled()) { -409 LOG.trace("No existing file in archive for: " + archiveFile + -410 ", free to archive original file."); -411 } -412 -413 // at this point, we should have a free spot for the archive file -414 boolean success = false; -415 for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) { -416 if (i > 0) { -417 // Ensure that the archive directory exists. -418 // The previous "move to archive" operation has failed probably because -419 // the cleaner has removed our archive directory (HBASE-7643). -420 // (we're in a retry loop, so don't worry too much about the exception) -421 try { -422 if (!fs.exists(archiveDir)) { -423 if (fs.mkdirs(archiveDir)) { -424 LOG.debug("Created archive directory:" + archiveDir); -425 } -426 } -427 } catch (IOException e) { -428 LOG.warn("Failed to create directory: " + archiveDir, e); -429 } -430 } -431 -432 try { -433 success = currentFile.moveAndClose(archiveFile); -434 } catch (FileNotFoundException fnfe) { -435 LOG.warn("Failed to archive " + currentFile + -436 " because it does not exist! Skipping and continuing on.", fnfe); -437 success = true; -438 } catch (IOException e) { -439 LOG.warn("Failed to archive " + currentFile + " on try #" + i, e); -440 success = false; -441 } +408 // at this point, we should have a free spot for the archive file +409 boolean success = false; +410 for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) { +411 if (i > 0) { +412 // Ensure that the archive directory exists. +413 // The previous "move to archive" operation has failed probably because +414 // the cleaner has removed our archive directory (HBASE-7643). +415 // (we're in a retry loop, so don't worry too much about the exception) +416 try { +417 if (!fs.exists(archiveDir)) { +418 if (fs.mkdirs(archiveDir)) { +419 LOG.debug("Created archive directory {}", archiveDir); +420 } +421 } +422 } catch (IOException e) { +423 LOG.warn("Failed to create directory {}", archiveDir, e); +424 } +425 } +426 +427 try { +428 success = currentFile.moveAndClose(archiveFile); +429 } catch (FileNotFoundException fnfe) { +430 LOG.warn("Failed to archive " + currentFile + +431 " because it does not exist! Skipping and continuing on.", fnfe); +432 success = true; +433 } catch (IOException e) { +434 LOG.warn("Failed to archive " + currentFile + " on try #" + i, e); +435 success = false; +436 } +437 } +438 +439 if (!success) { +440 LOG.error("Failed to archive " + currentFile); +441 return false; 442 } 443 -444 if (!success) { -445 LOG.error("Failed to archive " + currentFile); -446 return false; -447 } -448 -449 if (LOG.isDebugEnabled()) { -450 LOG.debug("Finished archiving from " + currentFile + ", to " + archiveFile); -451 } -452 return true; -453 } -454 -455 /** -456 * Without regard for backup, delete a region. Should be used with caution. -457 * @param regionDir {@link Path} to the region to be deleted. -458 * @param fs FileSystem from which to delete the region -459 * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise -460 * @throws IOException on filesystem operation failure -461 */ -462 private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir) -463 throws IOException { -464 if (fs.delete(regionDir, true)) { -465 LOG.debug("Deleted " + regionDir); -466 return true; -467 } -468 LOG.debug("Failed to delete region directory:" + regionDir); -469 return false; -470 } -471 -472 /** -473 * Just do a simple delete of the given store files -474 * <p> -475 * A best effort is made to delete each of the files, rather than bailing on the first failure. -476 * <p> -477 * @param compactedFiles store files to delete from the file system. -478 * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before -479 * throwing the exception, rather than failing at the first file. -480 */ -481 private static void deleteStoreFilesWithoutArchiving(Collection<HStoreFile> compactedFiles) -482 throws IOException { -483 LOG.debug("Deleting store files without archiving."); -484 List<IOException> errors = new ArrayList<>(0); -485 for (HStoreFile hsf : compactedFiles) { -486 try { -487 hsf.deleteStoreFile(); -488 } catch (IOException e) { -489 LOG.error("Failed to delete store file:" + hsf.getPath()); -490 errors.add(e); -491 } -492 } -493 if (errors.size() > 0) { -494 throw MultipleIOException.createIOException(errors); -495 } -496 } -497 -498 /** -499 * Adapt a type to match the {@link File} interface, which is used internally for handling -500 * archival/removal of files -501 * @param <T> type to adapt to the {@link File} interface -502 */ -503 private static abstract class FileConverter<T> implements Function<T, File> { -504 protected final FileSystem fs; -505 -506 public FileConverter(FileSystem fs) { -507 this.fs = fs; -508 } -509 } -510 -511 /** -512 * Convert a FileStatus to something we can manage in the archiving -513 */ -514 private static class FileStatusConverter extends FileConverter<FileStatus> { -515 public FileStatusConverter(FileSystem fs) { -516 super(fs); -517 } -518 -519 @Override -520 public File apply(FileStatus input) { -521 return new FileablePath(fs, input.getPath()); -522 } -523 } -524 -525 /** -526 * Convert the {@link HStoreFile} into something we can manage in the archive -527 * methods -528 */ -529 private static class StoreToFile extends FileConverter<HStoreFile> { -530 public StoreToFile(FileSystem fs) { -531 super(fs); -532 } -533 -534 @Override -535 public File apply(HStoreFile input) { -536 return new FileableStoreFile(fs, input); -537 } -538 } -539 -540 /** -541 * Wrapper to handle file operations uniformly -542 */ -543 private static abstract class File { -544 protected final FileSystem fs; -545 -546 public File(FileSystem fs) { -547 this.fs = fs; -548 } -549 -550 /** -551 * Delete the file -552 * @throws IOException on failure +444 LOG.debug("Archived from {} to {}", currentFile, archiveFile); +445 return true; +446 } +447 +448 /** +449 * Without regard for backup, delete a region. Should be used with caution. +450 * @param regionDir {@link Path} to the region to be deleted. +451 * @param fs FileSystem from which to delete the region +452 * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise +453 * @throws IOException on filesystem operation failure +454 */ +455 private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir) +456 throws IOException { +457 if (fs.delete(regionDir, true)) { +458 LOG.debug("Deleted {}", regionDir); +459 return true; +460 } +461 LOG.debug("Failed to delete directory {}", regionDir); +462 return false; +463 } +464 +465 /** +466 * Just do a simple delete of the given store files +467 * <p> +468 * A best effort is made to delete each of the files, rather than bailing on the first failure. +469 * <p> +470 * @param compactedFiles store files to delete from the file system. +471 * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before +472 * throwing the exception, rather than failing at the first file. +473 */ +474 private static void deleteStoreFilesWithoutArchiving(Collection<HStoreFile> compactedFiles) +475 throws IOException { +476 LOG.debug("Deleting files without archiving."); +477 List<IOException> errors = new ArrayList<>(0); +478 for (HStoreFile hsf : compactedFiles) { +479 try { +480 hsf.deleteStoreFile(); +481 } catch (IOException e) { +482 LOG.error("Failed to delete {}", hsf.getPath()); +483 errors.add(e); +484 } +485 } +486 if (errors.size() > 0) { +487 throw MultipleIOException.createIOException(errors); +488 } +489 } +490 +491 /** +492 * Adapt a type to match the {@link File} interface, which is used internally for handling +493 * archival/removal of files +494 * @param <T> type to adapt to the {@link File} interface +495 */ +496 private static abstract class FileConverter<T> implements Function<T, File> { +497 protected final FileSystem fs; +498 +499 public FileConverter(FileSystem fs) { +500 this.fs = fs; +501 } +502 } +503 +504 /** +505 * Convert a FileStatus to something we can manage in the archiving +506 */ +507 private static class FileStatusConverter extends FileConverter<FileStatus> { +508 public FileStatusConverter(FileSystem fs) { +509 super(fs); +510 } +511 +512 @Override +513 public File apply(FileStatus input) { +514 return new FileablePath(fs, input.getPath()); +515 } +516 } +517 +518 /** +519 * Convert the {@link HStoreFile} into something we can manage in the archive +520 * methods +521 */ +522 private static class StoreToFile extends FileConverter<HStoreFile> { +523 public StoreToFile(FileSystem fs) { +524 super(fs); +525 } +526 +527 @Override +528 public File apply(HStoreFile input) { +529 return new FileableStoreFile(fs, input); +530 } +531 } +532 +533 /** +534 * Wrapper to handle file operations uniformly +535 */ +536 private static abstract class File { +537 protected final FileSystem fs; +538 +539 public File(FileSystem fs) { +540 this.fs = fs; +541 } +542 +543 /** +544 * Delete the file +545 * @throws IOException on failure +546 */ +547 abstract void delete() throws IOException; +548 +549 /** +550 * Check to see if this is a file or a directory +551 * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise +552 * @throws IOException on {@link FileSystem} connection error 553 */ -554 abstract void delete() throws IOException; +554 abstract boolean isFile() throws IOException; 555 556 /** -557 * Check to see if this is a file or a directory -558 * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise -559 * @throws IOException on {@link FileSystem} connection error +557 * @return if this is a directory, returns all the children in the +558 * directory, otherwise returns an empty list +559 * @throws IOException 560 */ -561 abstract boolean isFile() throws IOException; +561 abstract Collection<File> getChildren() throws IOException; 562 563 /** -564 * @return if this is a directory, returns all the children in the -565 * directory, otherwise returns an empty list -566 * @throws IOException -567 */ -568 abstract Collection<File> getChildren() throws IOException; -569 -570 /** -571 * close any outside readers of the file -572 * @throws IOException -573 */ -574 abstract void close() throws IOException; -575 -576 /** -577 * @return the name of the file (not the full fs path, just the individual -578 * file name) -579 */ -580 abstract String getName(); -581 -582 /** -583 * @return the path to this file -584 */ -585 abstract Path getPath(); -586 -587 /** -588 * Move the file to the given destination -589 * @param dest -590 * @return <tt>true</tt> on success -591 * @throws IOException -592 */ -593 public boolean moveAndClose(Path dest) throws IOException { -594 this.close(); -595 Path p = this.getPath(); -596 return FSUtils.renameAndSetModifyTime(fs, p, dest); +564 * close any outside readers of the file +565 * @throws IOException +566 */ +567 abstract void close() throws IOException; +568 +569 /** +570 * @return the name of the file (not the full fs path, just the individual +571 * file name) +572 */ +573 abstract String getName(); +574 +575 /** +576 * @return the path to this file +577 */ +578 abstract Path getPath(); +579 +580 /** +581 * Move the file to the given destination +582