kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billy...@apache.org
Subject [04/11] kylin git commit: KYLIN-2291 Collect hive table property skip-header-line-count
Date Sun, 18 Dec 2016 06:15:47 GMT
KYLIN-2291 Collect hive table property skip-header-line-count

Signed-off-by: Li Yang <liyang@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d79d9374
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d79d9374
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d79d9374

Branch: refs/heads/KYLIN-2131
Commit: d79d9374cd8411518ee8eafc7640910826f6838c
Parents: f2377db
Author: Cheng Wang <cheng.wang@kyligence.io>
Authored: Fri Dec 16 17:23:15 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Fri Dec 16 17:45:16 2016 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/source/hive/BeelineHiveClient.java  |  3 +++
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java |  2 +-
 .../apache/kylin/source/hive/HiveSourceTableLoader.java  |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java |  4 +++-
 .../apache/kylin/source/hive/HiveTableMetaBuilder.java   | 11 ++++++++++-
 5 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 47b551b..468ccb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -193,6 +193,9 @@ public class BeelineHiveClient implements IHiveClient {
                     if ("numFiles".equals(resultSet.getString(2).trim())) {
                         builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
                     }
+                    if ("skip.header.line.count".equals(resultSet.getString(2).trim())) {
+                        builder.setSkipHeaderLineCount(resultSet.getString(3).trim());
+                    }
                 }
             }
             if ("InputFormat:".equals(resultSet.getString(1).trim())) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index 5a17f1f..e8a93bd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -42,7 +42,6 @@ import com.google.common.collect.Lists;
  *
  */
 public class CLIHiveClient implements IHiveClient {
-
     protected HiveConf hiveConf = null;
     protected Driver driver = null;
     protected HiveMetaStoreClient metaStoreClient = null;
@@ -109,6 +108,7 @@ public class CLIHiveClient implements IHiveClient {
         builder.setOwner(table.getOwner());
         builder.setLastAccessTime(table.getLastAccessTime());
         builder.setTableType(table.getTableType());
+        builder.setSkipHeaderLineCount(table.getParameters().get("skip.header.line.count"));
 
         return builder.createHiveTableMeta();
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 57292dc..77e1084 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -140,6 +140,7 @@ public class HiveSourceTableLoader {
             tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum));
             tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat);
             tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat);
+            tableExtDesc.addDataSourceProp("skip_header_line_count", String.valueOf(hiveTableMeta.skipHeaderLineCount));
 
             metaMgr.saveTableExt(tableExtDesc);
             metaMgr.saveSourceTable(tableDesc);

http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
index 784a0bb..fa9eb29 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -44,6 +44,7 @@ class HiveTableMeta {
     String sdOutputFormat;
     String owner;
     String tableType;
+    int skipHeaderLineCount;
     int lastAccessTime;
     long fileSize;
     long fileNum;
@@ -51,7 +52,7 @@ class HiveTableMeta {
     List<HiveTableColumnMeta> allColumns;
     List<HiveTableColumnMeta> partitionColumns;
 
-    public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String
sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum,
boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta>
partitionColumns) {
+    public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String
sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum,
int skipHeaderLineCount, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta>
partitionColumns) {
         this.tableName = tableName;
         this.sdLocation = sdLocation;
         this.sdInputFormat = sdInputFormat;
@@ -64,6 +65,7 @@ class HiveTableMeta {
         this.isNative = isNative;
         this.allColumns = allColumns;
         this.partitionColumns = partitionColumns;
+        this.skipHeaderLineCount = skipHeaderLineCount;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
index 7a3e5d6..073ded5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
@@ -32,6 +32,7 @@ public class HiveTableMetaBuilder {
     private int lastAccessTime;
     private long fileSize;
     private long fileNum;
+    private int skipHeaderLineCount;
     private boolean isNative = true;
     private List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
     private List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
@@ -81,6 +82,14 @@ public class HiveTableMetaBuilder {
         return this;
     }
 
+    public HiveTableMetaBuilder setSkipHeaderLineCount(String skipHeaderLineCount) {
+        if (null == skipHeaderLineCount)
+            this.skipHeaderLineCount = 0;
+        else
+            this.skipHeaderLineCount = Integer.parseInt(skipHeaderLineCount);
+        return this;
+    }
+
     public HiveTableMetaBuilder setIsNative(boolean isNative) {
         this.isNative = isNative;
         return this;
@@ -97,6 +106,6 @@ public class HiveTableMetaBuilder {
     }
 
     public HiveTableMeta createHiveTableMeta() {
-        return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner,
tableType, lastAccessTime, fileSize, fileNum, isNative, allColumns, partitionColumns);
+        return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner,
tableType, lastAccessTime, fileSize, fileNum, skipHeaderLineCount, isNative, allColumns, partitionColumns);
     }
 }
\ No newline at end of file


Mime
View raw message