hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anoopsamj...@apache.org
Subject hbase git commit: HBASE-14366 NPE in case visibility expression is not present in labels table during importtsv run.(Bhupendra)
Date Tue, 20 Oct 2015 11:05:29 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 76a14b9ba -> 834035bac


HBASE-14366 NPE in case visibility expression is not present in labels table during importtsv
run.(Bhupendra)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/834035ba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/834035ba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/834035ba

Branch: refs/heads/0.98
Commit: 834035bac118d45dd93ce6ae96372a0fe0f83a2e
Parents: 76a14b9
Author: anoopsjohn <anoopsamjohn@gmail.com>
Authored: Tue Oct 20 16:35:08 2015 +0530
Committer: anoopsjohn <anoopsamjohn@gmail.com>
Committed: Tue Oct 20 16:35:08 2015 +0530

----------------------------------------------------------------------
 .../visibility/VisibilityConstants.java         |  2 +
 .../DefaultVisibilityExpressionResolver.java    |  8 +-
 .../hadoop/hbase/mapreduce/TextSortReducer.java |  8 ++
 .../hbase/mapreduce/TsvImporterMapper.java      | 11 +++
 .../visibility/VisibilityLabelsCache.java       |  3 +-
 .../TestImportTSVWithVisibilityLabels.java      | 89 ++++++++++++++++++--
 6 files changed, 113 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/834035ba/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
index 89a94a9..7a0cea4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
@@ -58,4 +58,6 @@ public final class VisibilityConstants {
   public static final String OPEN_PARAN = "(";
   public static final String CLOSED_PARAN = ")";
 
+  /** Label ordinal value for invalid labels */
+  public static final int NON_EXIST_LABEL_ORDINAL = 0;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/834035ba/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
index 56eb1c1..1954f68 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.security.visibility.Authorizations;
+import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
 import org.apache.hadoop.hbase.security.visibility.VisibilityLabelOrdinalProvider;
 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -111,7 +112,12 @@ public class DefaultVisibilityExpressionResolver implements VisibilityExpression
     VisibilityLabelOrdinalProvider provider = new VisibilityLabelOrdinalProvider() {
       @Override
       public int getLabelOrdinal(String label) {
-        return labels.get(label);
+        Integer ordinal = null;
+        ordinal = labels.get(label);
+        if (ordinal != null) {
+          return ordinal.intValue();
+        }
+        return VisibilityConstants.NON_EXIST_LABEL_ORDINAL;
       }
 
       @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/834035ba/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TextSortReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TextSortReducer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TextSortReducer.java
index b3981a1..168ba40 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TextSortReducer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TextSortReducer.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.Tag;
 import org.apache.hadoop.hbase.TagType;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.visibility.InvalidLabelException;
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.Text;
@@ -184,6 +185,13 @@ public class TextSortReducer extends
             kvs.add(kv);
             curSize += kv.heapSize();
           }
+        } catch (InvalidLabelException badLine) {
+          if (skipBadLines) {
+            System.err.println("Bad line." + badLine.getMessage());
+            incrementBadLineCount(1);
+            continue;
+          }
+          throw new IOException(badLine);
         } catch (ImportTsv.TsvParser.BadTsvLineException badLine) {
           if (skipBadLines) {
             System.err.println("Bad line." + badLine.getMessage());

http://git-wip-us.apache.org/repos/asf/hbase/blob/834035ba/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
index 031efa0..efaa5c1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.BadTsvLineException;
 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
+import org.apache.hadoop.hbase.security.visibility.InvalidLabelException;
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.LongWritable;
@@ -164,6 +165,16 @@ extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put>
         populatePut(lineBytes, parsed, put, i);
       }
       context.write(rowKey, put);
+    } catch (InvalidLabelException badLine) {
+      if (skipBadLines) {
+        System.err.println(
+            "Bad line at offset: " + offset.get() + ":\n" +
+            badLine.getMessage());
+        incrementBadLineCount(1);
+        return;
+      } else {
+        throw new IOException(badLine);
+      }
     } catch (ImportTsv.TsvParser.BadTsvLineException badLine) {
       if (skipBadLines) {
         System.err.println(

http://git-wip-us.apache.org/repos/asf/hbase/blob/834035ba/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java
index 763a624..0948520 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java
@@ -49,7 +49,6 @@ import org.apache.zookeeper.KeeperException;
 public class VisibilityLabelsCache implements VisibilityLabelOrdinalProvider {
 
   private static final Log LOG = LogFactory.getLog(VisibilityLabelsCache.class);
-  private static final int NON_EXIST_LABEL_ORDINAL = 0;
   private static final List<String> EMPTY_LIST = Collections.emptyList();
   private static final Set<Integer> EMPTY_SET = Collections.emptySet();
   private static VisibilityLabelsCache instance;
@@ -175,7 +174,7 @@ public class VisibilityLabelsCache implements VisibilityLabelOrdinalProvider
{
       return ordinal.intValue();
     }
     // 0 denotes not available
-    return NON_EXIST_LABEL_ORDINAL;
+    return VisibilityConstants.NON_EXIST_LABEL_ORDINAL;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/834035ba/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
index a1557ed..851c9c4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
@@ -41,13 +41,15 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.security.visibility.Authorizations;
@@ -58,6 +60,7 @@ import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
 import org.apache.hadoop.hbase.security.visibility.VisibilityController;
 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapred.Utils.OutputFileUtils.OutputFilesFilter;
 import org.apache.hadoop.util.Tool;
@@ -268,6 +271,50 @@ public class TestImportTSVWithVisibilityLabels implements Configurable
{
     util.deleteTable(tableName);
   }
 
+  @Test
+  public void testBulkOutputWithInvalidLabels() throws Exception {
+    String tableName = "test-" + UUID.randomUUID();
+    Path hfiles = new Path(util.getDataTestDirOnTestFS(tableName), "hfiles");
+    // Prepare the arguments required for the test.
+    String[] args =
+        new String[] { "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=" + hfiles.toString(),
+            "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B,HBASE_CELL_VISIBILITY",
+            "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", tableName };
+
+    // 2 Data rows, one with valid label and one with invalid label
+    String data =
+        "KEY\u001bVALUE1\u001bVALUE2\u001bprivate\nKEY1\u001bVALUE1\u001bVALUE2\u001binvalid\n";
+    util.createTable(tableName, FAMILY);
+    doMROnTableTest(util, FAMILY, data, args, 1, 2);
+    util.deleteTable(tableName);
+  }
+
+  @Test
+  public void testBulkOutputWithTsvImporterTextMapperWithInvalidLabels() throws Exception
{
+    String tableName = "test-" + UUID.randomUUID();
+    Path hfiles = new Path(util.getDataTestDirOnTestFS(tableName), "hfiles");
+    // Prepare the arguments required for the test.
+    String[] args =
+        new String[] {
+            "-D" + ImportTsv.MAPPER_CONF_KEY
+                + "=org.apache.hadoop.hbase.mapreduce.TsvImporterTextMapper",
+            "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=" + hfiles.toString(),
+            "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B,HBASE_CELL_VISIBILITY",
+            "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", tableName };
+
+    // 2 Data rows, one with valid label and one with invalid label
+    String data =
+        "KEY\u001bVALUE1\u001bVALUE2\u001bprivate\nKEY1\u001bVALUE1\u001bVALUE2\u001binvalid\n";
+    util.createTable(tableName, FAMILY);
+    doMROnTableTest(util, FAMILY, data, args, 1, 2);
+    util.deleteTable(tableName);
+  }
+
+  protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
+      String[] args, int valueMultiplier) throws Exception {
+    return doMROnTableTest(util, family, data, args, valueMultiplier, -1);
+  }
+
   /**
    * Run an ImportTsv job and perform basic validation on the results. Returns
    * the ImportTsv <code>Tool</code> instance so that other tests can inspect
it
@@ -276,10 +323,12 @@ public class TestImportTSVWithVisibilityLabels implements Configurable
{
    *
    * @param args
    *          Any arguments to pass BEFORE inputFile path is appended.
+   * @param expectedKVCount Expected KV count. pass -1 to skip the kvcount check
+   *
    * @return The Tool instance used to run the test.
    */
   protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
-      String[] args, int valueMultiplier) throws Exception {
+      String[] args, int valueMultiplier, int expectedKVCount) throws Exception {
     String table = args[args.length - 1];
     Configuration conf = new Configuration(util.getConfiguration());
 
@@ -321,7 +370,7 @@ public class TestImportTSVWithVisibilityLabels implements Configurable
{
     }
     LOG.debug("validating the table " + createdHFiles);
     if (createdHFiles)
-     validateHFiles(fs, outputPath, family);
+     validateHFiles(fs, outputPath, family,expectedKVCount);
     else
       validateTable(conf, table, family, valueMultiplier);
 
@@ -335,14 +384,15 @@ public class TestImportTSVWithVisibilityLabels implements Configurable
{
   /**
    * Confirm ImportTsv via HFiles on fs.
    */
-  private static void validateHFiles(FileSystem fs, String outputPath, String family)
-      throws IOException {
+  private static void validateHFiles(FileSystem fs, String outputPath, String family,
+      int expectedKVCount) throws IOException {
 
     // validate number and content of output columns
     LOG.debug("Validating HFiles.");
     Set<String> configFamilies = new HashSet<String>();
     configFamilies.add(family);
     Set<String> foundFamilies = new HashSet<String>();
+    int actualKVCount = 0;
     for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter()))
{
       LOG.debug("The output path has files");
       String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);
@@ -354,8 +404,37 @@ public class TestImportTSVWithVisibilityLabels implements Configurable
{
       for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {
         assertTrue(String.format("HFile %s appears to contain no data.", hfile.getPath()),
             hfile.getLen() > 0);
+        if (expectedKVCount > -1) {
+          actualKVCount += getKVCountFromHfile(fs, hfile.getPath());
+        }
       }
     }
+    if (expectedKVCount > -1) {
+      assertTrue(String.format(
+        "KV count in output hfile=<%d> doesn't match with expected KV count=<%d>",
actualKVCount,
+        expectedKVCount), actualKVCount == expectedKVCount);
+    }
+  }
+
+  /**
+   * Method returns the total KVs in given hfile
+   * @param fs File System
+   * @param p HFile path
+   * @return KV count in the given hfile
+   * @throws IOException
+   */
+  private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
+    Configuration conf = util.getConfiguration();
+    HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
+    reader.loadFileInfo();
+    HFileScanner scanner = reader.getScanner(false, false);
+    scanner.seekTo();
+    int count = 0;
+    do {
+      count++;
+    } while (scanner.next());
+    reader.close();
+    return count;
   }
 
   /**


Mime
View raw message