hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-11329: Column prefix in key of hbase column prefix map (Wojciech Indyk, reviewed by Sergio Pena)
Date Tue, 08 Sep 2015 13:40:16 GMT
Repository: hive
Updated Branches:
  refs/heads/master 7281a4606 -> d51c62a45


HIVE-11329: Column prefix in key of hbase column prefix map (Wojciech Indyk, reviewed by Sergio
Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d51c62a4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d51c62a4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d51c62a4

Branch: refs/heads/master
Commit: d51c62a455eb08ee49f10ea2e117ca90de0bf47b
Parents: 7281a46
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Tue Sep 8 08:39:10 2015 -0500
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Tue Sep 8 08:39:10 2015 -0500

----------------------------------------------------------------------
 .../hadoop/hive/hbase/ColumnMappings.java       |  5 ++
 .../apache/hadoop/hive/hbase/HBaseSerDe.java    | 19 +++++-
 .../hadoop/hive/hbase/HBaseSerDeParameters.java |  8 ++-
 .../hadoop/hive/hbase/HBaseStorageHandler.java  | 13 ++--
 .../hadoop/hive/hbase/LazyHBaseCellMap.java     | 19 +++++-
 .../apache/hadoop/hive/hbase/LazyHBaseRow.java  |  5 +-
 .../hadoop/hive/hbase/LazyHBaseCellMapTest.java | 72 ++++++++++++++++++++
 .../positive/hbase_binary_map_queries_prefix.q  | 15 +++-
 .../hbase_binary_map_queries_prefix.q.out       | 40 +++++++++++
 9 files changed, 177 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java
index 5cb3752..d7d4461 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java
@@ -354,6 +354,7 @@ public class ColumnMappings implements Iterable<ColumnMappings.ColumnMapping>
{
     String mappingSpec;
     String qualifierPrefix;
     byte[] qualifierPrefixBytes;
+    boolean doPrefixCut;
 
     public String getColumnName() {
       return columnName;
@@ -399,6 +400,10 @@ public class ColumnMappings implements Iterable<ColumnMappings.ColumnMapping>
{
       return qualifierPrefixBytes;
     }
 
+    public boolean isDoPrefixCut(){
+      return doPrefixCut;
+    }
+
     public boolean isCategory(ObjectInspector.Category category) {
       return columnType.getCategory() == category;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
index e843bd8..ed2df5f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
@@ -93,6 +93,13 @@ public class HBaseSerDe extends AbstractSerDe {
    **/
   public static final String SERIALIZATION_TYPE = "serialization.type";
 
+  /**
+   * Defines if the prefix column from hbase should be hidden.
+   * It works only when @HBASE_COLUMNS_REGEX_MATCHING is true.
+   * Default value of this parameter is false
+   */
+  public static final String HBASE_COLUMNS_PREFIX_HIDE = "hbase.columns.mapping.prefix.hide";
+
   private ObjectInspector cachedObjectInspector;
   private LazyHBaseRow cachedHBaseRow;
 
@@ -136,6 +143,11 @@ public class HBaseSerDe extends AbstractSerDe {
       throws SerDeException {
     return parseColumnsMapping(columnsMappingSpec, true);
   }
+
+  public static ColumnMappings parseColumnsMapping(
+          String columnsMappingSpec, boolean doColumnRegexMatching) throws SerDeException
{
+	return parseColumnsMapping(columnsMappingSpec, doColumnRegexMatching, false);
+  }
   /**
    * Parses the HBase columns mapping specifier to identify the column families, qualifiers
    * and also caches the byte arrays corresponding to them. One of the Hive table
@@ -143,11 +155,12 @@ public class HBaseSerDe extends AbstractSerDe {
    *
    * @param columnsMappingSpec string hbase.columns.mapping specified when creating table
    * @param doColumnRegexMatching whether to do a regex matching on the columns or not
+   * @param hideColumnPrefix whether to hide a prefix of column mapping in key name in a
map (works only if @doColumnRegexMatching is true)
    * @return List<ColumnMapping> which contains the column mapping information by position
    * @throws org.apache.hadoop.hive.serde2.SerDeException
    */
   public static ColumnMappings parseColumnsMapping(
-      String columnsMappingSpec, boolean doColumnRegexMatching) throws SerDeException {
+      String columnsMappingSpec, boolean doColumnRegexMatching, boolean hideColumnPrefix)
throws SerDeException {
 
     if (columnsMappingSpec == null) {
       throw new SerDeException("Error: hbase.columns.mapping missing for this HBase table.");
@@ -206,6 +219,8 @@ public class HBaseSerDe extends AbstractSerDe {
             // we have a prefix with a wildcard
             columnMapping.qualifierPrefix = parts[1].substring(0, parts[1].length() - 2);
             columnMapping.qualifierPrefixBytes = Bytes.toBytes(columnMapping.qualifierPrefix);
+            //pass a flag to hide prefixes
+            columnMapping.doPrefixCut=hideColumnPrefix;
             // we weren't provided any actual qualifier name. Set these to
             // null.
             columnMapping.qualifierName = null;
@@ -214,6 +229,8 @@ public class HBaseSerDe extends AbstractSerDe {
             // set the regular provided qualifier names
             columnMapping.qualifierName = parts[1];
             columnMapping.qualifierNameBytes = Bytes.toBytes(parts[1]);
+            //if there is no prefix then we don't cut anything
+            columnMapping.doPrefixCut=false;
           }
         } else {
           columnMapping.qualifierName = null;

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
index 71f5da5..43c1f0c 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
@@ -22,6 +22,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
+import javax.annotation.Nullable;
+
 import org.apache.avro.Schema;
 import org.apache.avro.reflect.ReflectData;
 import org.apache.hadoop.conf.Configuration;
@@ -38,8 +40,6 @@ import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.util.ReflectionUtils;
 
-import javax.annotation.Nullable;
-
 /**
  * HBaseSerDeParameters encapsulates SerDeParameters and additional configurations that are
specific for
  * HBaseSerDe.
@@ -57,6 +57,7 @@ public class HBaseSerDeParameters {
   private final String columnMappingString;
   private final ColumnMappings columnMappings;
   private final boolean doColumnRegexMatching;
+  private final boolean doColumnPrefixCut;
 
   private final long putTimestamp;
   private final HBaseKeyFactory keyFactory;
@@ -69,8 +70,9 @@ public class HBaseSerDeParameters {
     columnMappingString = tbl.getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING);
     doColumnRegexMatching =
         Boolean.valueOf(tbl.getProperty(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, "true"));
+    doColumnPrefixCut = Boolean.valueOf(tbl.getProperty(HBaseSerDe.HBASE_COLUMNS_PREFIX_HIDE,
"false"));
     // Parse and initialize the HBase columns mapping
-    columnMappings = HBaseSerDe.parseColumnsMapping(columnMappingString, doColumnRegexMatching);
+    columnMappings = HBaseSerDe.parseColumnsMapping(columnMappingString, doColumnRegexMatching,
doColumnPrefixCut);
 
     // Build the type property string if not supplied
     String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
index c0eff23..cedb6e0 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
@@ -44,13 +44,10 @@ import org.apache.hadoop.hbase.mapred.TableOutputFormat;
 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
-import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector;
+import org.apache.hadoop.hbase.security.token.TokenUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
-import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.hive.metastore.HiveMetaHook;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -66,15 +63,11 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.hbase.security.token.TokenUtil;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.OutputFormat;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.zookeeper.KeeperException;
 
 import com.yammer.metrics.core.MetricsRegistry;
 
@@ -348,7 +341,9 @@ public class HBaseStorageHandler extends DefaultStorageHandler
       HBaseSerDe.HBASE_COLUMNS_MAPPING,
       tableProperties.getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING));
     jobProperties.put(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING,
-        tableProperties.getProperty(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, "true"));
+            tableProperties.getProperty(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, "true"));
+    jobProperties.put(HBaseSerDe.HBASE_COLUMNS_PREFIX_HIDE,
+            tableProperties.getProperty(HBaseSerDe.HBASE_COLUMNS_PREFIX_HIDE, "false"));
     jobProperties.put(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE,
       tableProperties.getProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE,"string"));
     String scanCache = tableProperties.getProperty(HBaseSerDe.HBASE_SCAN_CACHE);

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java
index 09cbf52..459d934 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java
@@ -45,8 +45,9 @@ public class LazyHBaseCellMap extends LazyMap {
   private byte [] columnFamilyBytes;
   private byte[] qualPrefix;
   private List<Boolean> binaryStorage;
+  private boolean hideQualPrefix;
 
-  /**
+	/**
    * Construct a LazyCellMap object with the ObjectInspector.
    * @param oi
    */
@@ -62,15 +63,23 @@ public class LazyHBaseCellMap extends LazyMap {
     init(r, columnFamilyBytes, binaryStorage, null);
   }
 
+	public void init(
+			Result r,
+			byte [] columnFamilyBytes,
+			List<Boolean> binaryStorage, byte[] qualPrefix) {
+		init(r, columnFamilyBytes, binaryStorage, qualPrefix, false);
+	}
+
   public void init(
       Result r,
       byte [] columnFamilyBytes,
-      List<Boolean> binaryStorage, byte[] qualPrefix) {
+      List<Boolean> binaryStorage, byte[] qualPrefix, boolean hideQualPrefix) {
     this.isNull = false;
     this.result = r;
     this.columnFamilyBytes = columnFamilyBytes;
     this.binaryStorage = binaryStorage;
     this.qualPrefix = qualPrefix;
+    this.hideQualPrefix = hideQualPrefix;
     setParsed(false);
   }
 
@@ -106,7 +115,11 @@ public class LazyHBaseCellMap extends LazyMap {
               binaryStorage.get(0));
 
         ByteArrayRef keyRef = new ByteArrayRef();
-        keyRef.setData(e.getKey());
+		  if (qualPrefix!=null && hideQualPrefix){
+			  keyRef.setData(Bytes.tail(e.getKey(), e.getKey().length-qualPrefix.length)); //cut prefix
from hive's map key
+		  }else{
+			  keyRef.setData(e.getKey()); //for non-prefix maps
+		  }
         key.init(keyRef, 0, keyRef.getData().length);
 
         // Value

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
index 868d81f..003dd1c 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
 import org.apache.hadoop.hive.hbase.struct.HBaseValueFactory;
@@ -36,6 +35,8 @@ import org.apache.hadoop.hive.serde2.lazy.LazyTimestamp;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * LazyObject for storing an HBase row.  The field of an HBase row can be
  * primitive or non-primitive.
@@ -148,7 +149,7 @@ public class LazyHBaseRow extends LazyStruct {
         // qualifier prefix to cherry pick the qualifiers that match the prefix instead of
picking
         // up everything
         ((LazyHBaseCellMap) fields[fieldID]).init(
-            result, colMap.familyNameBytes, colMap.binaryStorage, colMap.qualifierPrefixBytes);
+            result, colMap.familyNameBytes, colMap.binaryStorage, colMap.qualifierPrefixBytes,
colMap.isDoPrefixCut());
         return fields[fieldID].getObject();
       }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java
b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java
new file mode 100644
index 0000000..eb13e14
--- /dev/null
+++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class LazyHBaseCellMapTest extends TestCase {
+	public static final byte[] TEST_ROW = Bytes.toBytes("test-row");
+	public static final byte[] COLUMN_FAMILY = Bytes.toBytes("a");
+	public static final String QUAL_PREFIX = "col_";
+
+
+	public void testInitColumnPrefix() throws Exception {
+		Text nullSequence = new Text("\\N");
+		ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+				TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0),
+				new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
+
+		LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+
+		// Initialize a result
+		Cell[] cells = new KeyValue[2];
+
+		final String col1="1";
+		final String col2="2";
+		cells[0] = new KeyValue(TEST_ROW, COLUMN_FAMILY,
+				Bytes.toBytes(QUAL_PREFIX+col1), Bytes.toBytes("cfacol1"));
+		cells[1]=new KeyValue(TEST_ROW, COLUMN_FAMILY,
+				Bytes.toBytes(QUAL_PREFIX+col2), Bytes.toBytes("cfacol2"));
+
+		Result r = Result.create(cells);
+
+		List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
+		mapBinaryStorage.add(false);
+		mapBinaryStorage.add(false);
+
+		b.init(r, COLUMN_FAMILY, mapBinaryStorage, Bytes.toBytes(QUAL_PREFIX), true);
+
+		assertNotNull(b.getMapValueElement(new Text(col1)));
+		assertNotNull(b.getMapValueElement(new Text(col2)));
+
+	}
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/test/queries/positive/hbase_binary_map_queries_prefix.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_binary_map_queries_prefix.q b/hbase-handler/src/test/queries/positive/hbase_binary_map_queries_prefix.q
index 3d3f5ea..9ff4366 100644
--- a/hbase-handler/src/test/queries/positive/hbase_binary_map_queries_prefix.q
+++ b/hbase-handler/src/test/queries/positive/hbase_binary_map_queries_prefix.q
@@ -49,4 +49,17 @@ TBLPROPERTIES ("hbase.table.name"="t_hive_maps");
 
 SELECT * FROM t_ext_hbase_maps ORDER BY key;
 
-DROP TABLE t_ext_hbase_maps;
\ No newline at end of file
+DROP TABLE t_ext_hbase_maps;
+
+DROP TABLE t_ext_hbase_maps_cut_prefix;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps_cut_prefix(key STRING,
+                                       string_map_cols MAP<STRING, STRING>, simple_string_col
STRING)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-string:string_.*,cf-string:simple_string_col"
+    ,"hbase.columns.mapping.prefix.hide"="true")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps");
+
+SELECT * FROM t_ext_hbase_maps_cut_prefix ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_cut_prefix;

http://git-wip-us.apache.org/repos/asf/hive/blob/d51c62a4/hbase-handler/src/test/results/positive/hbase_binary_map_queries_prefix.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_binary_map_queries_prefix.q.out
b/hbase-handler/src/test/results/positive/hbase_binary_map_queries_prefix.q.out
index 5d21134..f6432b3 100644
--- a/hbase-handler/src/test/results/positive/hbase_binary_map_queries_prefix.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_binary_map_queries_prefix.q.out
@@ -152,3 +152,43 @@ POSTHOOK: query: DROP TABLE t_ext_hbase_maps
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@t_ext_hbase_maps
 POSTHOOK: Output: default@t_ext_hbase_maps
+PREHOOK: query: DROP TABLE t_ext_hbase_maps_cut_prefix
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE t_ext_hbase_maps_cut_prefix
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_maps_cut_prefix(key STRING,
+                                       string_map_cols MAP<STRING, STRING>, simple_string_col
STRING)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-string:string_.*,cf-string:simple_string_col"
+    ,"hbase.columns.mapping.prefix.hide"="true")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t_ext_hbase_maps_cut_prefix
+POSTHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_maps_cut_prefix(key STRING,
+                                       string_map_cols MAP<STRING, STRING>, simple_string_col
STRING)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-string:string_.*,cf-string:simple_string_col"
+    ,"hbase.columns.mapping.prefix.hide"="true")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t_ext_hbase_maps_cut_prefix
+PREHOOK: query: SELECT * FROM t_ext_hbase_maps_cut_prefix ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t_ext_hbase_maps_cut_prefix
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM t_ext_hbase_maps_cut_prefix ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t_ext_hbase_maps_cut_prefix
+#### A masked pattern was here ####
+125	{"col":"val_125"}	val_125
+126	{"col":"val_126"}	val_126
+PREHOOK: query: DROP TABLE t_ext_hbase_maps_cut_prefix
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t_ext_hbase_maps_cut_prefix
+PREHOOK: Output: default@t_ext_hbase_maps_cut_prefix
+POSTHOOK: query: DROP TABLE t_ext_hbase_maps_cut_prefix
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t_ext_hbase_maps_cut_prefix
+POSTHOOK: Output: default@t_ext_hbase_maps_cut_prefix


Mime
View raw message