Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0B7CF200CFC for ; Thu, 28 Sep 2017 17:14:24 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 09F961609EE; Thu, 28 Sep 2017 15:14:24 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C51F71609C2 for ; Thu, 28 Sep 2017 17:14:22 +0200 (CEST) Received: (qmail 77016 invoked by uid 500); 28 Sep 2017 15:14:13 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 76164 invoked by uid 99); 28 Sep 2017 15:14:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Sep 2017 15:14:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A036BF5BDF; Thu, 28 Sep 2017 15:14:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Thu, 28 Sep 2017 15:14:42 -0000 Message-Id: In-Reply-To: <0ae50d3876544e89813c25027ef0caaa@git.apache.org> References: <0ae50d3876544e89813c25027ef0caaa@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [34/51] [partial] hbase-site git commit: Published site at . archived-at: Thu, 28 Sep 2017 15:14:24 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/67deb422/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.html b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.html index 38d76b6..5478df1 100644 --- a/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.html +++ b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.html @@ -90,115 +90,115 @@ 082import org.apache.hadoop.io.SequenceFile; 083import org.apache.hadoop.io.Text; 084import org.apache.hadoop.mapreduce.Job; -085import org.apache.hadoop.mapreduce.OutputFormat; -086import org.apache.hadoop.mapreduce.RecordWriter; -087import org.apache.hadoop.mapreduce.TaskAttemptContext; -088import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; -089import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -090import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner; -091import org.apache.yetus.audience.InterfaceAudience; -092 -093import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; -094 -095/** -096 * Writes HFiles. Passed Cells must arrive in order. -097 * Writes current time as the sequence id for the file. Sets the major compacted -098 * attribute on created @{link {@link HFile}s. Calling write(null,null) will forcibly roll -099 * all HFiles being written. -100 * -101 * Using this class as part of a MapReduce job is best done -102 * using {@link #configureIncrementalLoad(Job, TableDescriptor, RegionLocator)}. -103 */ -104@InterfaceAudience.Public -105public class HFileOutputFormat2 -106 extends FileOutputFormat<ImmutableBytesWritable, Cell> { -107 private static final Log LOG = LogFactory.getLog(HFileOutputFormat2.class); -108 static class TableInfo { -109 private TableDescriptor tableDesctiptor; -110 private RegionLocator regionLocator; -111 -112 public TableInfo(TableDescriptor tableDesctiptor, RegionLocator regionLocator) { -113 this.tableDesctiptor = tableDesctiptor; -114 this.regionLocator = regionLocator; -115 } -116 -117 /** -118 * The modification for the returned HTD doesn't affect the inner TD. -119 * @return A clone of inner table descriptor -120 * @deprecated use {@link #getTableDescriptor} -121 */ -122 @Deprecated -123 public HTableDescriptor getHTableDescriptor() { -124 return new HTableDescriptor(tableDesctiptor); -125 } -126 -127 public TableDescriptor getTableDescriptor() { -128 return tableDesctiptor; -129 } -130 -131 public RegionLocator getRegionLocator() { -132 return regionLocator; -133 } -134 } -135 -136 protected static final byte[] tableSeparator = ";".getBytes(StandardCharsets.UTF_8); -137 -138 protected static byte[] combineTableNameSuffix(byte[] tableName, -139 byte[] suffix ) { -140 return Bytes.add(tableName, tableSeparator, suffix); -141 } -142 -143 // The following constants are private since these are used by -144 // HFileOutputFormat2 to internally transfer data between job setup and -145 // reducer run using conf. -146 // These should not be changed by the client. -147 static final String COMPRESSION_FAMILIES_CONF_KEY = -148 "hbase.hfileoutputformat.families.compression"; -149 static final String BLOOM_TYPE_FAMILIES_CONF_KEY = -150 "hbase.hfileoutputformat.families.bloomtype"; -151 static final String BLOCK_SIZE_FAMILIES_CONF_KEY = -152 "hbase.mapreduce.hfileoutputformat.blocksize"; -153 static final String DATABLOCK_ENCODING_FAMILIES_CONF_KEY = -154 "hbase.mapreduce.hfileoutputformat.families.datablock.encoding"; -155 -156 // This constant is public since the client can modify this when setting -157 // up their conf object and thus refer to this symbol. -158 // It is present for backwards compatibility reasons. Use it only to -159 // override the auto-detection of datablock encoding. -160 public static final String DATABLOCK_ENCODING_OVERRIDE_CONF_KEY = -161 "hbase.mapreduce.hfileoutputformat.datablock.encoding"; -162 -163 /** -164 * Keep locality while generating HFiles for bulkload. See HBASE-12596 -165 */ -166 public static final String LOCALITY_SENSITIVE_CONF_KEY = -167 "hbase.bulkload.locality.sensitive.enabled"; -168 private static final boolean DEFAULT_LOCALITY_SENSITIVE = true; -169 static final String OUTPUT_TABLE_NAME_CONF_KEY = -170 "hbase.mapreduce.hfileoutputformat.table.name"; -171 static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY = -172 "hbase.mapreduce.use.multi.table.hfileoutputformat"; -173 -174 public static final String STORAGE_POLICY_PROPERTY = "hbase.hstore.storagepolicy"; -175 public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; -176 -177 @Override -178 public RecordWriter<ImmutableBytesWritable, Cell> getRecordWriter( -179 final TaskAttemptContext context) throws IOException, InterruptedException { -180 return createRecordWriter(context); -181 } -182 -183 protected static byte[] getTableNameSuffixedWithFamily(byte[] tableName, byte[] family) { -184 return combineTableNameSuffix(tableName, family); -185 } -186 -187 static <V extends Cell> RecordWriter<ImmutableBytesWritable, V> -188 createRecordWriter(final TaskAttemptContext context) -189 throws IOException { -190 -191 // Get the path of the temporary output file -192 final Path outputPath = FileOutputFormat.getOutputPath(context); -193 final Path outputDir = new FileOutputCommitter(outputPath, context).getWorkPath(); +085import org.apache.hadoop.mapreduce.OutputCommitter; +086import org.apache.hadoop.mapreduce.OutputFormat; +087import org.apache.hadoop.mapreduce.RecordWriter; +088import org.apache.hadoop.mapreduce.TaskAttemptContext; +089import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +090import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +091import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner; +092import org.apache.yetus.audience.InterfaceAudience; +093 +094import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; +095 +096/** +097 * Writes HFiles. Passed Cells must arrive in order. +098 * Writes current time as the sequence id for the file. Sets the major compacted +099 * attribute on created @{link {@link HFile}s. Calling write(null,null) will forcibly roll +100 * all HFiles being written. +101 * +102 * Using this class as part of a MapReduce job is best done +103 * using {@link #configureIncrementalLoad(Job, TableDescriptor, RegionLocator)}. +104 */ +105@InterfaceAudience.Public +106public class HFileOutputFormat2 +107 extends FileOutputFormat<ImmutableBytesWritable, Cell> { +108 private static final Log LOG = LogFactory.getLog(HFileOutputFormat2.class); +109 static class TableInfo { +110 private TableDescriptor tableDesctiptor; +111 private RegionLocator regionLocator; +112 +113 public TableInfo(TableDescriptor tableDesctiptor, RegionLocator regionLocator) { +114 this.tableDesctiptor = tableDesctiptor; +115 this.regionLocator = regionLocator; +116 } +117 +118 /** +119 * The modification for the returned HTD doesn't affect the inner TD. +120 * @return A clone of inner table descriptor +121 * @deprecated use {@link #getTableDescriptor} +122 */ +123 @Deprecated +124 public HTableDescriptor getHTableDescriptor() { +125 return new HTableDescriptor(tableDesctiptor); +126 } +127 +128 public TableDescriptor getTableDescriptor() { +129 return tableDesctiptor; +130 } +131 +132 public RegionLocator getRegionLocator() { +133 return regionLocator; +134 } +135 } +136 +137 protected static final byte[] tableSeparator = ";".getBytes(StandardCharsets.UTF_8); +138 +139 protected static byte[] combineTableNameSuffix(byte[] tableName, +140 byte[] suffix ) { +141 return Bytes.add(tableName, tableSeparator, suffix); +142 } +143 +144 // The following constants are private since these are used by +145 // HFileOutputFormat2 to internally transfer data between job setup and +146 // reducer run using conf. +147 // These should not be changed by the client. +148 static final String COMPRESSION_FAMILIES_CONF_KEY = +149 "hbase.hfileoutputformat.families.compression"; +150 static final String BLOOM_TYPE_FAMILIES_CONF_KEY = +151 "hbase.hfileoutputformat.families.bloomtype"; +152 static final String BLOCK_SIZE_FAMILIES_CONF_KEY = +153 "hbase.mapreduce.hfileoutputformat.blocksize"; +154 static final String DATABLOCK_ENCODING_FAMILIES_CONF_KEY = +155 "hbase.mapreduce.hfileoutputformat.families.datablock.encoding"; +156 +157 // This constant is public since the client can modify this when setting +158 // up their conf object and thus refer to this symbol. +159 // It is present for backwards compatibility reasons. Use it only to +160 // override the auto-detection of datablock encoding. +161 public static final String DATABLOCK_ENCODING_OVERRIDE_CONF_KEY = +162 "hbase.mapreduce.hfileoutputformat.datablock.encoding"; +163 +164 /** +165 * Keep locality while generating HFiles for bulkload. See HBASE-12596 +166 */ +167 public static final String LOCALITY_SENSITIVE_CONF_KEY = +168 "hbase.bulkload.locality.sensitive.enabled"; +169 private static final boolean DEFAULT_LOCALITY_SENSITIVE = true; +170 static final String OUTPUT_TABLE_NAME_CONF_KEY = +171 "hbase.mapreduce.hfileoutputformat.table.name"; +172 static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY = +173 "hbase.mapreduce.use.multi.table.hfileoutputformat"; +174 +175 public static final String STORAGE_POLICY_PROPERTY = "hbase.hstore.storagepolicy"; +176 public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; +177 +178 @Override +179 public RecordWriter<ImmutableBytesWritable, Cell> getRecordWriter( +180 final TaskAttemptContext context) throws IOException, InterruptedException { +181 return createRecordWriter(context, this.getOutputCommitter(context)); +182 } +183 +184 protected static byte[] getTableNameSuffixedWithFamily(byte[] tableName, byte[] family) { +185 return combineTableNameSuffix(tableName, family); +186 } +187 +188 static <V extends Cell> RecordWriter<ImmutableBytesWritable, V> +189 createRecordWriter(final TaskAttemptContext context, final OutputCommitter committer) +190 throws IOException { +191 +192 // Get the path of the temporary output file +193 final Path outputDir = ((FileOutputCommitter)committer).getWorkPath(); 194 final Configuration conf = context.getConfiguration(); 195 final boolean writeMultipleTables = conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false) ; 196 final String writeTableNames = conf.get(OUTPUT_TABLE_NAME_CONF_KEY); http://git-wip-us.apache.org/repos/asf/hbase-site/blob/67deb422/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.html b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.html index bb1b9b6..2ff9932 100644 --- a/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.html +++ b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.html @@ -26,195 +26,205 @@ 018 019package org.apache.hadoop.hbase.mapreduce; 020 -021import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; -022import org.apache.hadoop.fs.Path; -023import org.apache.hadoop.hbase.HRegionInfo; -024import org.apache.hadoop.hbase.HTableDescriptor; -025import org.apache.yetus.audience.InterfaceAudience; -026import org.apache.hadoop.hbase.client.Result; -027import org.apache.hadoop.hbase.client.Scan; -028import org.apache.hadoop.hbase.client.metrics.ScanMetrics; -029import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -030import org.apache.hadoop.io.Writable; -031import org.apache.hadoop.mapreduce.InputFormat; -032import org.apache.hadoop.mapreduce.InputSplit; -033import org.apache.hadoop.mapreduce.Job; -034import org.apache.hadoop.mapreduce.JobContext; -035import org.apache.hadoop.mapreduce.RecordReader; -036import org.apache.hadoop.mapreduce.TaskAttemptContext; -037 -038import java.io.DataInput; -039import java.io.DataOutput; -040import java.io.IOException; -041import java.lang.reflect.Method; -042import java.util.ArrayList; -043import java.util.List; +021import java.io.DataInput; +022import java.io.DataOutput; +023import java.io.IOException; +024import java.lang.reflect.Method; +025import java.util.ArrayList; +026import java.util.List; +027 +028import org.apache.hadoop.fs.Path; +029import org.apache.hadoop.hbase.HRegionInfo; +030import org.apache.hadoop.hbase.HTableDescriptor; +031import org.apache.hadoop.hbase.client.RegionInfo; +032import org.apache.hadoop.hbase.client.Result; +033import org.apache.hadoop.hbase.client.Scan; +034import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +035import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +036import org.apache.hadoop.io.Writable; +037import org.apache.hadoop.mapreduce.InputFormat; +038import org.apache.hadoop.mapreduce.InputSplit; +039import org.apache.hadoop.mapreduce.Job; +040import org.apache.hadoop.mapreduce.JobContext; +041import org.apache.hadoop.mapreduce.RecordReader; +042import org.apache.hadoop.mapreduce.TaskAttemptContext; +043import org.apache.yetus.audience.InterfaceAudience; 044 -045/** -046 * TableSnapshotInputFormat allows a MapReduce job to run over a table snapshot. The job -047 * bypasses HBase servers, and directly accesses the underlying files (hfile, recovered edits, -048 * wals, etc) directly to provide maximum performance. The snapshot is not required to be -049 * restored to the live cluster or cloned. This also allows to run the mapreduce job from an -050 * online or offline hbase cluster. The snapshot files can be exported by using the -051 * {@link org.apache.hadoop.hbase.snapshot.ExportSnapshot} tool, to a pure-hdfs cluster, -052 * and this InputFormat can be used to run the mapreduce job directly over the snapshot files. -053 * The snapshot should not be deleted while there are jobs reading from snapshot files. -054 * -055 * Usage is similar to TableInputFormat, and -056 * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job, boolean, Path)} -057 * can be used to configure the job. -058 * <pre>{@code -059 * Job job = new Job(conf); -060 * Scan scan = new Scan(); -061 * TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, -062 * scan, MyTableMapper.class, MyMapKeyOutput.class, -063 * MyMapOutputValueWritable.class, job, true); -064 * } -065 * </pre> -066 * -067 * Internally, this input format restores the snapshot into the given tmp directory. Similar to -068 * {@link TableInputFormat} an InputSplit is created per region. The region is opened for reading -069 * from each RecordReader. An internal RegionScanner is used to execute the -070 * {@link org.apache.hadoop.hbase.CellScanner} obtained from the user. -071 * -072 * HBase owns all the data and snapshot files on the filesystem. Only the 'hbase' user can read from -073 * snapshot files and data files. -074 * To read from snapshot files directly from the file system, the user who is running the MR job -075 * must have sufficient permissions to access snapshot and reference files. -076 * This means that to run mapreduce over snapshot files, the MR job has to be run as the HBase -077 * user or the user must have group or other privileges in the filesystem (See HBASE-8369). -078 * Note that, given other users access to read from snapshot/data files will completely circumvent -079 * the access control enforced by HBase. -080 * @see org.apache.hadoop.hbase.client.TableSnapshotScanner -081 */ -082@InterfaceAudience.Public -083public class TableSnapshotInputFormat extends InputFormat<ImmutableBytesWritable, Result> { -084 -085 public static class TableSnapshotRegionSplit extends InputSplit implements Writable { -086 private TableSnapshotInputFormatImpl.InputSplit delegate; -087 -088 // constructor for mapreduce framework / Writable -089 public TableSnapshotRegionSplit() { -090 this.delegate = new TableSnapshotInputFormatImpl.InputSplit(); -091 } -092 -093 public TableSnapshotRegionSplit(TableSnapshotInputFormatImpl.InputSplit delegate) { -094 this.delegate = delegate; -095 } -096 -097 public TableSnapshotRegionSplit(HTableDescriptor htd, HRegionInfo regionInfo, -098 List<String> locations, Scan scan, Path restoreDir) { -099 this.delegate = -100 new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations, scan, restoreDir); -101 } -102 -103 @Override -104 public long getLength() throws IOException, InterruptedException { -105 return delegate.getLength(); -106 } -107 -108 @Override -109 public String[] getLocations() throws IOException, InterruptedException { -110 return delegate.getLocations(); -111 } -112 -113 @Override -114 public void write(DataOutput out) throws IOException { -115 delegate.write(out); -116 } -117 -118 @Override -119 public void readFields(DataInput in) throws IOException { -120 delegate.readFields(in); -121 } -122 -123 public HRegionInfo getRegionInfo() { -124 return delegate.getRegionInfo(); -125 } -126 -127 } -128 -129 @VisibleForTesting -130 static class TableSnapshotRegionRecordReader extends -131 RecordReader<ImmutableBytesWritable, Result> { -132 private TableSnapshotInputFormatImpl.RecordReader delegate = -133 new TableSnapshotInputFormatImpl.RecordReader(); -134 private TaskAttemptContext context; -135 private Method getCounter; -136 -137 @Override -138 public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, -139 InterruptedException { -140 this.context = context; -141 getCounter = TableRecordReaderImpl.retrieveGetCounterWithStringsParams(context); -142 delegate.initialize( -143 ((TableSnapshotRegionSplit) split).delegate, -144 context.getConfiguration()); -145 } +045import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; +046 +047/** +048 * TableSnapshotInputFormat allows a MapReduce job to run over a table snapshot. The job +049 * bypasses HBase servers, and directly accesses the underlying files (hfile, recovered edits, +050 * wals, etc) directly to provide maximum performance. The snapshot is not required to be +051 * restored to the live cluster or cloned. This also allows to run the mapreduce job from an +052 * online or offline hbase cluster. The snapshot files can be exported by using the +053 * {@link org.apache.hadoop.hbase.snapshot.ExportSnapshot} tool, to a pure-hdfs cluster, +054 * and this InputFormat can be used to run the mapreduce job directly over the snapshot files. +055 * The snapshot should not be deleted while there are jobs reading from snapshot files. +056 * +057 * Usage is similar to TableInputFormat, and +058 * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job, boolean, Path)} +059 * can be used to configure the job. +060 * <pre>{@code +061 * Job job = new Job(conf); +062 * Scan scan = new Scan(); +063 * TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, +064 * scan, MyTableMapper.class, MyMapKeyOutput.class, +065 * MyMapOutputValueWritable.class, job, true); +066 * } +067 * </pre> +068 * +069 * Internally, this input format restores the snapshot into the given tmp directory. Similar to +070 * {@link TableInputFormat} an InputSplit is created per region. The region is opened for reading +071 * from each RecordReader. An internal RegionScanner is used to execute the +072 * {@link org.apache.hadoop.hbase.CellScanner} obtained from the user. +073 * +074 * HBase owns all the data and snapshot files on the filesystem. Only the 'hbase' user can read from +075 * snapshot files and data files. +076 * To read from snapshot files directly from the file system, the user who is running the MR job +077 * must have sufficient permissions to access snapshot and reference files. +078 * This means that to run mapreduce over snapshot files, the MR job has to be run as the HBase +079 * user or the user must have group or other privileges in the filesystem (See HBASE-8369). +080 * Note that, given other users access to read from snapshot/data files will completely circumvent +081 * the access control enforced by HBase. +082 * @see org.apache.hadoop.hbase.client.TableSnapshotScanner +083 */ +084@InterfaceAudience.Public +085public class TableSnapshotInputFormat extends InputFormat<ImmutableBytesWritable, Result> { +086 +087 public static class TableSnapshotRegionSplit extends InputSplit implements Writable { +088 private TableSnapshotInputFormatImpl.InputSplit delegate; +089 +090 // constructor for mapreduce framework / Writable +091 public TableSnapshotRegionSplit() { +092 this.delegate = new TableSnapshotInputFormatImpl.InputSplit(); +093 } +094 +095 public TableSnapshotRegionSplit(TableSnapshotInputFormatImpl.InputSplit delegate) { +096 this.delegate = delegate; +097 } +098 +099 public TableSnapshotRegionSplit(HTableDescriptor htd, HRegionInfo regionInfo, +100 List<String> locations, Scan scan, Path restoreDir) { +101 this.delegate = +102 new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations, scan, restoreDir); +103 } +104 +105 @Override +106 public long getLength() throws IOException, InterruptedException { +107 return delegate.getLength(); +108 } +109 +110 @Override +111 public String[] getLocations() throws IOException, InterruptedException { +112 return delegate.getLocations(); +113 } +114 +115 @Override +116 public void write(DataOutput out) throws IOException { +117 delegate.write(out); +118 } +119 +120 @Override +121 public void readFields(DataInput in) throws IOException { +122 delegate.readFields(in); +123 } +124 +125 /** +126 * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0 +127 * Use {@link #getRegion()} +128 */ +129 @Deprecated +130 public HRegionInfo getRegionInfo() { +131 return delegate.getRegionInfo(); +132 } +133 +134 public RegionInfo getRegion() { +135 return delegate.getRegionInfo(); +136 } +137 } +138 +139 @VisibleForTesting +140 static class TableSnapshotRegionRecordReader extends +141 RecordReader<ImmutableBytesWritable, Result> { +142 private TableSnapshotInputFormatImpl.RecordReader delegate = +143 new TableSnapshotInputFormatImpl.RecordReader(); +144 private TaskAttemptContext context; +145 private Method getCounter; 146 147 @Override -148 public boolean nextKeyValue() throws IOException, InterruptedException { -149 boolean result = delegate.nextKeyValue(); -150 if (result) { -151 ScanMetrics scanMetrics = delegate.getScanner().getScanMetrics(); -152 if (scanMetrics != null && context != null) { -153 TableRecordReaderImpl.updateCounters(scanMetrics, 0, getCounter, context, 0); -154 } -155 } -156 return result; -157 } -158 -159 @Override -160 public ImmutableBytesWritable getCurrentKey() throws IOException, InterruptedException { -161 return delegate.getCurrentKey(); -162 } -163 -164 @Override -165 public Result getCurrentValue() throws IOException, InterruptedException { -166 return delegate.getCurrentValue(); +148 public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, +149 InterruptedException { +150 this.context = context; +151 getCounter = TableRecordReaderImpl.retrieveGetCounterWithStringsParams(context); +152 delegate.initialize( +153 ((TableSnapshotRegionSplit) split).delegate, +154 context.getConfiguration()); +155 } +156 +157 @Override +158 public boolean nextKeyValue() throws IOException, InterruptedException { +159 boolean result = delegate.nextKeyValue(); +160 if (result) { +161 ScanMetrics scanMetrics = delegate.getScanner().getScanMetrics(); +162 if (scanMetrics != null && context != null) { +163 TableRecordReaderImpl.updateCounters(scanMetrics, 0, getCounter, context, 0); +164 } +165 } +166 return result; 167 } 168 169 @Override -170 public float getProgress() throws IOException, InterruptedException { -171 return delegate.getProgress(); +170 public ImmutableBytesWritable getCurrentKey() throws IOException, InterruptedException { +171 return delegate.getCurrentKey(); 172 } 173 174 @Override -175 public void close() throws IOException { -176 delegate.close(); +175 public Result getCurrentValue() throws IOException, InterruptedException { +176 return delegate.getCurrentValue(); 177 } -178 } -179 -180 @Override -181 public RecordReader<ImmutableBytesWritable, Result> createRecordReader( -182 InputSplit split, TaskAttemptContext context) throws IOException { -183 return new TableSnapshotRegionRecordReader(); -184 } -185 -186 @Override -187 public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { -188 List<InputSplit> results = new ArrayList<>(); -189 for (TableSnapshotInputFormatImpl.InputSplit split : -190 TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) { -191 results.add(new TableSnapshotRegionSplit(split)); -192 } -193 return results; +178 +179 @Override +180 public float getProgress() throws IOException, InterruptedException { +181 return delegate.getProgress(); +182 } +183 +184 @Override +185 public void close() throws IOException { +186 delegate.close(); +187 } +188 } +189 +190 @Override +191 public RecordReader<ImmutableBytesWritable, Result> createRecordReader( +192 InputSplit split, TaskAttemptContext context) throws IOException { +193 return new TableSnapshotRegionRecordReader(); 194 } 195 -196 /** -197 * Configures the job to use TableSnapshotInputFormat to read from a snapshot. -198 * @param job the job to configure -199 * @param snapshotName the name of the snapshot to read from -200 * @param restoreDir a temporary directory to restore the snapshot into. Current user should -201 * have write permissions to this directory, and this should not be a subdirectory of rootdir. -202 * After the job is finished, restoreDir can be deleted. -203 * @throws IOException if an error occurs -204 */ -205 public static void setInput(Job job, String snapshotName, Path restoreDir) -206 throws IOException { -207 TableSnapshotInputFormatImpl.setInput(job.getConfiguration(), snapshotName, restoreDir); -208 } -209} +196 @Override +197 public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { +198 List<InputSplit> results = new ArrayList<>(); +199 for (TableSnapshotInputFormatImpl.InputSplit split : +200 TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) { +201 results.add(new TableSnapshotRegionSplit(split)); +202 } +203 return results; +204 } +205 +206 /** +207 * Configures the job to use TableSnapshotInputFormat to read from a snapshot. +208 * @param job the job to configure +209 * @param snapshotName the name of the snapshot to read from +210 * @param restoreDir a temporary directory to restore the snapshot into. Current user should +211 * have write permissions to this directory, and this should not be a subdirectory of rootdir. +212 * After the job is finished, restoreDir can be deleted. +213 * @throws IOException if an error occurs +214 */ +215 public static void setInput(Job job, String snapshotName, Path restoreDir) +216 throws IOException { +217 TableSnapshotInputFormatImpl.setInput(job.getConfiguration(), snapshotName, restoreDir); +218 } +219}