kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qhz...@apache.org
Subject [51/52] [abbrv] incubator-kylin git commit: KYLIN-883 Using configurable option for Hive intermediate tables
Date Mon, 27 Jul 2015 09:21:53 GMT
KYLIN-883 Using configurable option for Hive intermediate tables 


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/39566040
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/39566040
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/39566040

Branch: refs/heads/0.8
Commit: 39566040870050b356146a2aead43858c8de2eee
Parents: 3d04704
Author: shaofengshi <shaofengshi@apache.org>
Authored: Mon Jul 27 14:47:43 2015 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Mon Jul 27 14:47:43 2015 +0800

----------------------------------------------------------------------
 conf/kylin.properties                                         | 3 +++
 .../src/main/java/org/apache/kylin/common/KylinConfig.java    | 7 +++++++
 .../org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java  | 6 ++++--
 .../main/java/org/apache/kylin/source/hive/HiveMRInput.java   | 5 ++++-
 4 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39566040/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index e55f49d..7da9a9d 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -36,6 +36,9 @@ kylin.job.concurrent.max.limit=10
 # Time interval to check hadoop job status
 kylin.job.yarn.app.rest.check.interval.seconds=10
 
+# Hive database name for putting the intermediate flat tables
+kylin.job.hive.database.for.intermediatetable=default
+
 # Whether calculate cube in mem in each mapper;
 kylin.job.cubing.inMem=true
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39566040/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 89a7fcc..1352214 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -99,6 +99,8 @@ public class KylinConfig {
 
     public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
 
+    public static final String HIVE_DATABASE_FOR_INTERMEDIATE_TABLE = "kylin.job.hive.database.for.intermediatetable";
+    
     public static final String HIVE_PASSWORD = "hive.password";
 
     public static final String HIVE_USER = "hive.user";
@@ -657,6 +659,11 @@ public class KylinConfig {
         this.storageUrl = storageUrl;
     }
 
+
+    public String getHiveDatabaseForIntermediateTable() {
+        return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, "default");
+    }
+
     public void setRunAsRemoteCommand(String v) {
         kylinConfig.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v);
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39566040/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
index 62dd343..c82ceb6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
@@ -20,6 +20,7 @@ package org.apache.kylin.engine.mr.steps;
 
 import com.google.common.collect.Lists;
 
+import com.google.common.collect.Sets;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
@@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Set;
 
 /**
  */
@@ -48,7 +50,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
 
     private KylinConfig config = null;
 
-    public static final long TIME_THREADSHOLD = 2 * 26 * 3600 * 1000l; // 2 days
+    public static final long TIME_THREADSHOLD = 2 * 24 * 3600 * 1000l; // 2 days
 
     /*
      * (non-Javadoc)
@@ -94,7 +96,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
     public void cleanup() throws Exception {
         CubeManager cubeManager = CubeManager.getInstance(config);
 
-        List<String> activeResourceList = Lists.newArrayList();
+        Set<String> activeResourceList = Sets.newHashSet();
         for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
             for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
                 activeResourceList.addAll(segment.getSnapshotPaths());

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39566040/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index c40924e..28e23b9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -101,6 +101,7 @@ public class HiveMRInput implements IMRInput {
         
         public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc
flatTableDesc, String jobId) {
 
+            final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable()
+ ";";
             final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
             final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc,
JobBuilderSupport.getJobWorkingDir(conf, jobId));
             String insertDataHqls;
@@ -147,7 +148,9 @@ public class HiveMRInput implements IMRInput {
 
             final String hiveTable = this.getOldHiveTable();
             if (StringUtils.isNotEmpty(hiveTable)) {
-                final String dropHiveCMD = "hive -e \"DROP TABLE IF EXISTS  " + hiveTable
+ ";\"";
+                final String dropSQL = "USE " + context.getConfig().getHiveDatabaseForIntermediateTable()
+ ";"
+                        + " DROP TABLE IF EXISTS  " + hiveTable + ";";
+                final String dropHiveCMD = "hive -e \"" + dropSQL + "\"";
                 ShellCmdOutput shellCmdOutput = new ShellCmdOutput();
                 try {
                     context.getConfig().getCliCommandExecutor().execute(dropHiveCMD, shellCmdOutput);


Mime
View raw message