Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 475F5200D90 for ; Sat, 30 Dec 2017 16:19:02 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 45B5E160C11; Sat, 30 Dec 2017 15:19:02 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0D33B160C3D for ; Sat, 30 Dec 2017 16:18:59 +0100 (CET) Received: (qmail 76415 invoked by uid 500); 30 Dec 2017 15:18:53 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 75011 invoked by uid 99); 30 Dec 2017 15:18:52 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 30 Dec 2017 15:18:52 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EC25AE9640; Sat, 30 Dec 2017 15:18:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 30 Dec 2017 15:19:13 -0000 Message-Id: In-Reply-To: <8891f102c9ed4b5c8ec80f3146f95a47@git.apache.org> References: <8891f102c9ed4b5c8ec80f3146f95a47@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [28/51] [partial] hbase-site git commit: Published site at . archived-at: Sat, 30 Dec 2017 15:19:02 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/83bf6175/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionTool.CompactionInputFormat.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionTool.CompactionInputFormat.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionTool.CompactionInputFormat.html index 91eec45..d1cd185 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionTool.CompactionInputFormat.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionTool.CompactionInputFormat.html @@ -88,400 +88,396 @@ 080 private final static String CONF_COMPACT_ONCE = "hbase.compactiontool.compact.once"; 081 private final static String CONF_COMPACT_MAJOR = "hbase.compactiontool.compact.major"; 082 private final static String CONF_DELETE_COMPACTED = "hbase.compactiontool.delete"; -083 private final static String CONF_COMPLETE_COMPACTION = "hbase.hstore.compaction.complete"; -084 -085 /** -086 * Class responsible to execute the Compaction on the specified path. -087 * The path can be a table, region or family directory. -088 */ -089 private static class CompactionWorker { -090 private final boolean keepCompactedFiles; -091 private final boolean deleteCompacted; -092 private final Configuration conf; -093 private final FileSystem fs; -094 private final Path tmpDir; -095 -096 public CompactionWorker(final FileSystem fs, final Configuration conf) { -097 this.conf = conf; -098 this.keepCompactedFiles = !conf.getBoolean(CONF_COMPLETE_COMPACTION, true); -099 this.deleteCompacted = conf.getBoolean(CONF_DELETE_COMPACTED, false); -100 this.tmpDir = new Path(conf.get(CONF_TMP_DIR)); -101 this.fs = fs; -102 } -103 -104 /** -105 * Execute the compaction on the specified path. -106 * -107 * @param path Directory path on which to run compaction. -108 * @param compactOnce Execute just a single step of compaction. -109 * @param major Request major compaction. -110 */ -111 public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException { -112 if (isFamilyDir(fs, path)) { -113 Path regionDir = path.getParent(); -114 Path tableDir = regionDir.getParent(); -115 TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); -116 RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); -117 compactStoreFiles(tableDir, htd, hri, -118 path.getName(), compactOnce, major); -119 } else if (isRegionDir(fs, path)) { -120 Path tableDir = path.getParent(); -121 TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); -122 compactRegion(tableDir, htd, path, compactOnce, major); -123 } else if (isTableDir(fs, path)) { -124 compactTable(path, compactOnce, major); -125 } else { -126 throw new IOException( -127 "Specified path is not a table, region or family directory. path=" + path); -128 } -129 } -130 -131 private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major) -132 throws IOException { -133 TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); -134 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { -135 compactRegion(tableDir, htd, regionDir, compactOnce, major); -136 } -137 } -138 -139 private void compactRegion(final Path tableDir, final TableDescriptor htd, -140 final Path regionDir, final boolean compactOnce, final boolean major) -141 throws IOException { -142 RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); -143 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { -144 compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); -145 } -146 } -147 -148 /** -149 * Execute the actual compaction job. -150 * If the compact once flag is not specified, execute the compaction until -151 * no more compactions are needed. Uses the Configuration settings provided. -152 */ -153 private void compactStoreFiles(final Path tableDir, final TableDescriptor htd, -154 final RegionInfo hri, final String familyName, final boolean compactOnce, -155 final boolean major) throws IOException { -156 HStore store = getStore(conf, fs, tableDir, htd, hri, familyName, tmpDir); -157 LOG.info("Compact table=" + htd.getTableName() + -158 " region=" + hri.getRegionNameAsString() + -159 " family=" + familyName); -160 if (major) { -161 store.triggerMajorCompaction(); -162 } -163 do { -164 Optional<CompactionContext> compaction = -165 store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); -166 if (!compaction.isPresent()) { -167 break; -168 } -169 List<HStoreFile> storeFiles = -170 store.compact(compaction.get(), NoLimitThroughputController.INSTANCE, null); -171 if (storeFiles != null && !storeFiles.isEmpty()) { -172 if (keepCompactedFiles && deleteCompacted) { -173 for (HStoreFile storeFile: storeFiles) { -174 fs.delete(storeFile.getPath(), false); -175 } -176 } -177 } -178 } while (store.needsCompaction() && !compactOnce); -179 } -180 -181 /** -182 * Create a "mock" HStore that uses the tmpDir specified by the user and -183 * the store dir to compact as source. -184 */ -185 private static HStore getStore(final Configuration conf, final FileSystem fs, -186 final Path tableDir, final TableDescriptor htd, final RegionInfo hri, -187 final String familyName, final Path tempDir) throws IOException { -188 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) { -189 @Override -190 public Path getTempDir() { -191 return tempDir; -192 } -193 }; -194 HRegion region = new HRegion(regionFs, null, conf, htd, null); -195 return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf); -196 } -197 } -198 -199 private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException { -200 Path regionInfo = new Path(path, HRegionFileSystem.REGION_INFO_FILE); -201 return fs.exists(regionInfo); -202 } -203 -204 private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException { -205 return FSTableDescriptors.getTableInfoPath(fs, path) != null; -206 } -207 -208 private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException { -209 return isRegionDir(fs, path.getParent()); -210 } -211 -212 private static class CompactionMapper -213 extends Mapper<LongWritable, Text, NullWritable, NullWritable> { -214 private CompactionWorker compactor = null; -215 private boolean compactOnce = false; -216 private boolean major = false; -217 -218 @Override -219 public void setup(Context context) { -220 Configuration conf = context.getConfiguration(); -221 compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false); -222 major = conf.getBoolean(CONF_COMPACT_MAJOR, false); -223 -224 try { -225 FileSystem fs = FileSystem.get(conf); -226 this.compactor = new CompactionWorker(fs, conf); -227 } catch (IOException e) { -228 throw new RuntimeException("Could not get the input FileSystem", e); -229 } -230 } -231 -232 @Override -233 public void map(LongWritable key, Text value, Context context) -234 throws InterruptedException, IOException { -235 Path path = new Path(value.toString()); -236 this.compactor.compact(path, compactOnce, major); -237 } -238 } -239 -240 /** -241 * Input format that uses store files block location as input split locality. -242 */ -243 private static class CompactionInputFormat extends TextInputFormat { -244 @Override -245 protected boolean isSplitable(JobContext context, Path file) { -246 return true; -247 } -248 -249 /** -250 * Returns a split for each store files directory using the block location -251 * of each file as locality reference. -252 */ -253 @Override -254 public List<InputSplit> getSplits(JobContext job) throws IOException { -255 List<InputSplit> splits = new ArrayList<>(); -256 List<FileStatus> files = listStatus(job); -257 -258 Text key = new Text(); -259 for (FileStatus file: files) { -260 Path path = file.getPath(); -261 FileSystem fs = path.getFileSystem(job.getConfiguration()); -262 LineReader reader = new LineReader(fs.open(path)); -263 long pos = 0; -264 int n; -265 try { -266 while ((n = reader.readLine(key)) > 0) { -267 String[] hosts = getStoreDirHosts(fs, path); -268 splits.add(new FileSplit(path, pos, n, hosts)); -269 pos += n; -270 } -271 } finally { -272 reader.close(); -273 } -274 } +083 +084 /** +085 * Class responsible to execute the Compaction on the specified path. +086 * The path can be a table, region or family directory. +087 */ +088 private static class CompactionWorker { +089 private final boolean deleteCompacted; +090 private final Configuration conf; +091 private final FileSystem fs; +092 private final Path tmpDir; +093 +094 public CompactionWorker(final FileSystem fs, final Configuration conf) { +095 this.conf = conf; +096 this.deleteCompacted = conf.getBoolean(CONF_DELETE_COMPACTED, false); +097 this.tmpDir = new Path(conf.get(CONF_TMP_DIR)); +098 this.fs = fs; +099 } +100 +101 /** +102 * Execute the compaction on the specified path. +103 * +104 * @param path Directory path on which to run compaction. +105 * @param compactOnce Execute just a single step of compaction. +106 * @param major Request major compaction. +107 */ +108 public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException { +109 if (isFamilyDir(fs, path)) { +110 Path regionDir = path.getParent(); +111 Path tableDir = regionDir.getParent(); +112 TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); +113 RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); +114 compactStoreFiles(tableDir, htd, hri, +115 path.getName(), compactOnce, major); +116 } else if (isRegionDir(fs, path)) { +117 Path tableDir = path.getParent(); +118 TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); +119 compactRegion(tableDir, htd, path, compactOnce, major); +120 } else if (isTableDir(fs, path)) { +121 compactTable(path, compactOnce, major); +122 } else { +123 throw new IOException( +124 "Specified path is not a table, region or family directory. path=" + path); +125 } +126 } +127 +128 private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major) +129 throws IOException { +130 TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); +131 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { +132 compactRegion(tableDir, htd, regionDir, compactOnce, major); +133 } +134 } +135 +136 private void compactRegion(final Path tableDir, final TableDescriptor htd, +137 final Path regionDir, final boolean compactOnce, final boolean major) +138 throws IOException { +139 RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); +140 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { +141 compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); +142 } +143 } +144 +145 /** +146 * Execute the actual compaction job. +147 * If the compact once flag is not specified, execute the compaction until +148 * no more compactions are needed. Uses the Configuration settings provided. +149 */ +150 private void compactStoreFiles(final Path tableDir, final TableDescriptor htd, +151 final RegionInfo hri, final String familyName, final boolean compactOnce, +152 final boolean major) throws IOException { +153 HStore store = getStore(conf, fs, tableDir, htd, hri, familyName, tmpDir); +154 LOG.info("Compact table=" + htd.getTableName() + +155 " region=" + hri.getRegionNameAsString() + +156 " family=" + familyName); +157 if (major) { +158 store.triggerMajorCompaction(); +159 } +160 do { +161 Optional<CompactionContext> compaction = +162 store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); +163 if (!compaction.isPresent()) { +164 break; +165 } +166 List<HStoreFile> storeFiles = +167 store.compact(compaction.get(), NoLimitThroughputController.INSTANCE, null); +168 if (storeFiles != null && !storeFiles.isEmpty()) { +169 if (deleteCompacted) { +170 for (HStoreFile storeFile: storeFiles) { +171 fs.delete(storeFile.getPath(), false); +172 } +173 } +174 } +175 } while (store.needsCompaction() && !compactOnce); +176 } +177 +178 /** +179 * Create a "mock" HStore that uses the tmpDir specified by the user and +180 * the store dir to compact as source. +181 */ +182 private static HStore getStore(final Configuration conf, final FileSystem fs, +183 final Path tableDir, final TableDescriptor htd, final RegionInfo hri, +184 final String familyName, final Path tempDir) throws IOException { +185 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) { +186 @Override +187 public Path getTempDir() { +188 return tempDir; +189 } +190 }; +191 HRegion region = new HRegion(regionFs, null, conf, htd, null); +192 return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf); +193 } +194 } +195 +196 private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException { +197 Path regionInfo = new Path(path, HRegionFileSystem.REGION_INFO_FILE); +198 return fs.exists(regionInfo); +199 } +200 +201 private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException { +202 return FSTableDescriptors.getTableInfoPath(fs, path) != null; +203 } +204 +205 private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException { +206 return isRegionDir(fs, path.getParent()); +207 } +208 +209 private static class CompactionMapper +210 extends Mapper<LongWritable, Text, NullWritable, NullWritable> { +211 private CompactionWorker compactor = null; +212 private boolean compactOnce = false; +213 private boolean major = false; +214 +215 @Override +216 public void setup(Context context) { +217 Configuration conf = context.getConfiguration(); +218 compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false); +219 major = conf.getBoolean(CONF_COMPACT_MAJOR, false); +220 +221 try { +222 FileSystem fs = FileSystem.get(conf); +223 this.compactor = new CompactionWorker(fs, conf); +224 } catch (IOException e) { +225 throw new RuntimeException("Could not get the input FileSystem", e); +226 } +227 } +228 +229 @Override +230 public void map(LongWritable key, Text value, Context context) +231 throws InterruptedException, IOException { +232 Path path = new Path(value.toString()); +233 this.compactor.compact(path, compactOnce, major); +234 } +235 } +236 +237 /** +238 * Input format that uses store files block location as input split locality. +239 */ +240 private static class CompactionInputFormat extends TextInputFormat { +241 @Override +242 protected boolean isSplitable(JobContext context, Path file) { +243 return true; +244 } +245 +246 /** +247 * Returns a split for each store files directory using the block location +248 * of each file as locality reference. +249 */ +250 @Override +251 public List<InputSplit> getSplits(JobContext job) throws IOException { +252 List<InputSplit> splits = new ArrayList<>(); +253 List<FileStatus> files = listStatus(job); +254 +255 Text key = new Text(); +256 for (FileStatus file: files) { +257 Path path = file.getPath(); +258 FileSystem fs = path.getFileSystem(job.getConfiguration()); +259 LineReader reader = new LineReader(fs.open(path)); +260 long pos = 0; +261 int n; +262 try { +263 while ((n = reader.readLine(key)) > 0) { +264 String[] hosts = getStoreDirHosts(fs, path); +265 splits.add(new FileSplit(path, pos, n, hosts)); +266 pos += n; +267 } +268 } finally { +269 reader.close(); +270 } +271 } +272 +273 return splits; +274 } 275 -276 return splits; -277 } -278 -279 /** -280 * return the top hosts of the store files, used by the Split -281 */ -282 private static String[] getStoreDirHosts(final FileSystem fs, final Path path) -283 throws IOException { -284 FileStatus[] files = FSUtils.listStatus(fs, path); -285 if (files == null) { -286 return new String[] {}; -287 } -288 -289 HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); -290 for (FileStatus hfileStatus: files) { -291 HDFSBlocksDistribution storeFileBlocksDistribution = -292 FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); -293 hdfsBlocksDistribution.add(storeFileBlocksDistribution); -294 } -295 -296 List<String> hosts = hdfsBlocksDistribution.getTopHosts(); -297 return hosts.toArray(new String[hosts.size()]); -298 } -299 -300 /** -301 * Create the input file for the given directories to compact. -302 * The file is a TextFile with each line corrisponding to a -303 * store files directory to compact. -304 */ -305 public static void createInputFile(final FileSystem fs, final Path path, -306 final Set<Path> toCompactDirs) throws IOException { -307 // Extract the list of store dirs -308 List<Path> storeDirs = new LinkedList<>(); -309 for (Path compactDir: toCompactDirs) { -310 if (isFamilyDir(fs, compactDir)) { -311 storeDirs.add(compactDir); -312 } else if (isRegionDir(fs, compactDir)) { -313 for (Path familyDir: FSUtils.getFamilyDirs(fs, compactDir)) { -314 storeDirs.add(familyDir); -315 } -316 } else if (isTableDir(fs, compactDir)) { -317 // Lookup regions -318 for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) { -319 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { -320 storeDirs.add(familyDir); -321 } -322 } -323 } else { -324 throw new IOException( -325 "Specified path is not a table, region or family directory. path=" + compactDir); -326 } -327 } -328 -329 // Write Input File -330 FSDataOutputStream stream = fs.create(path); -331 LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact."); -332 try { -333 final byte[] newLine = Bytes.toBytes("\n"); -334 for (Path storeDir: storeDirs) { -335 stream.write(Bytes.toBytes(storeDir.toString())); -336 stream.write(newLine); -337 } -338 } finally { -339 stream.close(); -340 } -341 } -342 } -343 -344 /** -345 * Execute compaction, using a Map-Reduce job. -346 */ -347 private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, -348 final boolean compactOnce, final boolean major) throws Exception { -349 Configuration conf = getConf(); -350 conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); -351 conf.setBoolean(CONF_COMPACT_MAJOR, major); -352 -353 Job job = new Job(conf); -354 job.setJobName("CompactionTool"); -355 job.setJarByClass(CompactionTool.class); -356 job.setMapperClass(CompactionMapper.class); -357 job.setInputFormatClass(CompactionInputFormat.class); -358 job.setOutputFormatClass(NullOutputFormat.class); -359 job.setMapSpeculativeExecution(false); -360 job.setNumReduceTasks(0); +276 /** +277 * return the top hosts of the store files, used by the Split +278 */ +279 private static String[] getStoreDirHosts(final FileSystem fs, final Path path) +280 throws IOException { +281 FileStatus[] files = FSUtils.listStatus(fs, path); +282 if (files == null) { +283 return new String[] {}; +284 } +285 +286 HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); +287 for (FileStatus hfileStatus: files) { +288 HDFSBlocksDistribution storeFileBlocksDistribution = +289 FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); +290 hdfsBlocksDistribution.add(storeFileBlocksDistribution); +291 } +292 +293 List<String> hosts = hdfsBlocksDistribution.getTopHosts(); +294 return hosts.toArray(new String[hosts.size()]); +295 } +296 +297 /** +298 * Create the input file for the given directories to compact. +299 * The file is a TextFile with each line corrisponding to a +300 * store files directory to compact. +301 */ +302 public static void createInputFile(final FileSystem fs, final Path path, +303 final Set<Path> toCompactDirs) throws IOException { +304 // Extract the list of store dirs +305 List<Path> storeDirs = new LinkedList<>(); +306 for (Path compactDir: toCompactDirs) { +307 if (isFamilyDir(fs, compactDir)) { +308 storeDirs.add(compactDir); +309 } else if (isRegionDir(fs, compactDir)) { +310 for (Path familyDir: FSUtils.getFamilyDirs(fs, compactDir)) { +311 storeDirs.add(familyDir); +312 } +313 } else if (isTableDir(fs, compactDir)) { +314 // Lookup regions +315 for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) { +316 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { +317 storeDirs.add(familyDir); +318 } +319 } +320 } else { +321 throw new IOException( +322 "Specified path is not a table, region or family directory. path=" + compactDir); +323 } +324 } +325 +326 // Write Input File +327 FSDataOutputStream stream = fs.create(path); +328 LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact."); +329 try { +330 final byte[] newLine = Bytes.toBytes("\n"); +331 for (Path storeDir: storeDirs) { +332 stream.write(Bytes.toBytes(storeDir.toString())); +333 stream.write(newLine); +334 } +335 } finally { +336 stream.close(); +337 } +338 } +339 } +340 +341 /** +342 * Execute compaction, using a Map-Reduce job. +343 */ +344 private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, +345 final boolean compactOnce, final boolean major) throws Exception { +346 Configuration conf = getConf(); +347 conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); +348 conf.setBoolean(CONF_COMPACT_MAJOR, major); +349 +350 Job job = new Job(conf); +351 job.setJobName("CompactionTool"); +352 job.setJarByClass(CompactionTool.class); +353 job.setMapperClass(CompactionMapper.class); +354 job.setInputFormatClass(CompactionInputFormat.class); +355 job.setOutputFormatClass(NullOutputFormat.class); +356 job.setMapSpeculativeExecution(false); +357 job.setNumReduceTasks(0); +358 +359 // add dependencies (including HBase ones) +360 TableMapReduceUtil.addDependencyJars(job); 361 -362 // add dependencies (including HBase ones) -363 TableMapReduceUtil.addDependencyJars(job); -364 -365 Path stagingDir = JobUtil.getStagingDir(conf); -366 try { -367 // Create input file with the store dirs -368 Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTime()); -369 CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); -370 CompactionInputFormat.addInputPath(job, inputPath); +362 Path stagingDir = JobUtil.getStagingDir(conf); +363 try { +364 // Create input file with the store dirs +365 Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTime()); +366 CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); +367 CompactionInputFormat.addInputPath(job, inputPath); +368 +369 // Initialize credential for secure cluster +370 TableMapReduceUtil.initCredentials(job); 371 -372 // Initialize credential for secure cluster -373 TableMapReduceUtil.initCredentials(job); -374 -375 // Start the MR Job and wait -376 return job.waitForCompletion(true) ? 0 : 1; -377 } finally { -378 fs.delete(stagingDir, true); -379 } -380 } -381 -382 /** -383 * Execute compaction, from this client, one path at the time. -384 */ -385 private int doClient(final FileSystem fs, final Set<Path> toCompactDirs, -386 final boolean compactOnce, final boolean major) throws IOException { -387 CompactionWorker worker = new CompactionWorker(fs, getConf()); -388 for (Path path: toCompactDirs) { -389 worker.compact(path, compactOnce, major); -390 } -391 return 0; -392 } -393 -394 @Override -395 public int run(String[] args) throws Exception { -396 Set<Path> toCompactDirs = new HashSet<>(); -397 boolean compactOnce = false; -398 boolean major = false; -399 boolean mapred = false; +372 // Start the MR Job and wait +373 return job.waitForCompletion(true) ? 0 : 1; +374 } finally { +375 fs.delete(stagingDir, true); +376 } +377 } +378 +379 /** +380 * Execute compaction, from this client, one path at the time. +381 */ +382 private int doClient(final FileSystem fs, final Set<Path> toCompactDirs, +383 final boolean compactOnce, final boolean major) throws IOException { +384 CompactionWorker worker = new CompactionWorker(fs, getConf()); +385 for (Path path: toCompactDirs) { +386 worker.compact(path, compactOnce, major); +387 } +388 return 0; +389 } +390 +391 @Override +392 public int run(String[] args) throws Exception { +393 Set<Path> toCompactDirs = new HashSet<>(); +394 boolean compactOnce = false; +395 boolean major = false; +396 boolean mapred = false; +397 +398 Configuration conf = getConf(); +399 FileSystem fs = FileSystem.get(conf); 400 -401 Configuration conf = getConf(); -402 FileSystem fs = FileSystem.get(conf); -403 -404 try { -405 for (int i = 0; i < args.length; ++i) { -406 String opt = args[i]; -407 if (opt.equals("-compactOnce")) { -408 compactOnce = true; -409 } else if (opt.equals("-major")) { -410 major = true; -411 } else if (opt.equals("-mapred")) { -412 mapred = true; -413 } else if (!opt.startsWith("-")) { -414 Path path = new Path(opt); -415 FileStatus status = fs.getFileStatus(path); -416 if (!status.isDirectory()) { -417 printUsage("Specified path is not a directory. path=" + path); -418 return 1; -419 } -420 toCompactDirs.add(path); -421 } else { -422 printUsage(); -423 } -424 } -425 } catch (Exception e) { -426 printUsage(e.getMessage()); -427 return 1; -428 } -429 -430 if (toCompactDirs.isEmpty()) { -431 printUsage("No directories to compact specified."); -432 return 1; -433 } -434 -435 // Execute compaction! -436 if (mapred) { -437 return doMapReduce(fs, toCompactDirs, compactOnce, major); -438 } else { -439 return doClient(fs, toCompactDirs, compactOnce, major); -440 } -441 } -442 -443 private void printUsage() { -444 printUsage(null); -445 } -446 -447 private void printUsage(final String message) { -448 if (message != null && message.length() > 0) { -449 System.err.println(message); -450 } -451 System.err.println("Usage: java " + this.getClass().getName() + " \\"); -452 System.err.println(" [-compactOnce] [-major] [-mapred] [-D<property=value>]* files..."); -453 System.err.println(); -454 System.err.println("Options:"); -455 System.err.println(" mapred Use MapReduce to run compaction."); -456 System.err.println(" compactOnce Execute just one compaction step. (default: while needed)"); -457 System.err.println(" major Trigger major compaction."); -458 System.err.println(); -459 System.err.println("Note: -D properties will be applied to the conf used. "); -460 System.err.println("For example: "); -461 System.err.println(" To preserve input files, pass -D"+CONF_COMPLETE_COMPACTION+"=false"); -462 System.err.println(" To stop delete of compacted file, pass -D"+CONF_DELETE_COMPACTED+"=false"); -463 System.err.println(" To set tmp dir, pass -D"+CONF_TMP_DIR+"=ALTERNATE_DIR"); +401 try { +402 for (int i = 0; i < args.length; ++i) { +403 String opt = args[i]; +404 if (opt.equals("-compactOnce")) { +405 compactOnce = true; +406 } else if (opt.equals("-major")) { +407 major = true; +408 } else if (opt.equals("-mapred")) { +409 mapred = true; +410 } else if (!opt.startsWith("-")) { +411 Path path = new Path(opt); +412 FileStatus status = fs.getFileStatus(path); +413 if (!status.isDirectory()) { +414 printUsage("Specified path is not a directory. path=" + path); +415 return 1; +416 } +417 toCompactDirs.add(path); +418 } else { +419 printUsage(); +420 } +421 } +422 } catch (Exception e) { +423 printUsage(e.getMessage()); +424 return 1; +425 } +426 +427 if (toCompactDirs.isEmpty()) { +428 printUsage("No directories to compact specified."); +429 return 1; +430 } +431 +432 // Execute compaction! +433 if (mapred) { +434 return doMapReduce(fs, toCompactDirs, compactOnce, major); +435 } else { +436 return doClient(fs, toCompactDirs, compactOnce, major); +437 } +438 } +439 +440 private void printUsage() { +441 printUsage(null); +442 } +443 +444 private void printUsage(final String message) { +445 if (message != null && message.length() > 0) { +446 System.err.println(message); +447 } +448 System.err.println("Usage: java " + this.getClass().getName() + " \\"); +449 System.err.println(" [-compactOnce] [-major] [-mapred] [-D<property=value>]* files..."); +450 System.err.println(); +451 System.err.println("Options:"); +452 System.err.println(" mapred Use MapReduce to run compaction."); +453 System.err.println(" compactOnce Execute just one compaction step. (default: while needed)"); +454 System.err.println(" major Trigger major compaction."); +455 System.err.println(); +456 System.err.println("Note: -D properties will be applied to the conf used. "); +457 System.err.println("For example: "); +458 System.err.println(" To stop delete of compacted file, pass -D"+CONF_DELETE_COMPACTED+"=false"); +459 System.err.println(" To set tmp dir, pass -D"+CONF_TMP_DIR+"=ALTERNATE_DIR"); +460 System.err.println(); +461 System.err.println("Examples:"); +462 System.err.println(" To compact the full 'TestTable' using MapReduce:"); +463 System.err.println(" $ hbase " + this.getClass().getName() + " -mapred hdfs:///hbase/data/default/TestTable"); 464 System.err.println(); -465 System.err.println("Examples:"); -466 System.err.println(" To compact the full 'TestTable' using MapReduce:"); -467 System.err.println(" $ hbase " + this.getClass().getName() + " -mapred hdfs:///hbase/data/default/TestTable"); -468 System.err.println(); -469 System.err.println(" To compact column family 'x' of the table 'TestTable' region 'abc':"); -470 System.err.println(" $ hbase " + this.getClass().getName() + " hdfs:///hbase/data/default/TestTable/abc/x"); +465 System.err.println(" To compact column family 'x' of the table 'TestTable' region 'abc':"); +466 System.err.println(" $ hbase " + this.getClass().getName() + " hdfs:///hbase/data/default/TestTable/abc/x"); +467 } +468 +469 public static void main(String[] args) throws Exception { +470 System.exit(ToolRunner.run(HBaseConfiguration.create(), new CompactionTool(), args)); 471 } -472 -473 public static void main(String[] args) throws Exception { -474 System.exit(ToolRunner.run(HBaseConfiguration.create(), new CompactionTool(), args)); -475 } -476} +472}