kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [2/2] kylin git commit: KYLIN-1614 Job diagnosis
Date Thu, 28 Apr 2016 11:19:30 GMT
KYLIN-1614 Job diagnosis


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6a5c187b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6a5c187b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6a5c187b

Branch: refs/heads/master
Commit: 6a5c187bd962d38e6839353b20106584d8a04107
Parents: 35bd664
Author: lidongsjtu <lidong@apache.org>
Authored: Thu Apr 28 14:04:21 2016 +0800
Committer: lidongsjtu <lidong@apache.org>
Committed: Thu Apr 28 18:54:18 2016 +0800

----------------------------------------------------------------------
 build/bin/kylin.sh                              |  6 +--
 .../org/apache/kylin/common/KylinVersion.java   |  4 +-
 .../java/org/apache/kylin/cube/CubeManager.java |  2 +
 .../java/org/apache/kylin/cube/CubeSegment.java |  8 ++-
 .../kylin/storage/hbase/HBaseConnection.java    |  5 ++
 tool/pom.xml                                    | 56 ++++++++++++++------
 .../apache/kylin/tool/CubeMetaExtractor.java    |  3 --
 .../org/apache/kylin/tool/DiagnosisInfoCLI.java | 24 +++++++--
 .../apache/kylin/tool/HBaseUsageExtractor.java  |  3 --
 .../org/apache/kylin/tool/JobInfoExtractor.java | 34 +++++++-----
 10 files changed, 101 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index af90e4c..34014d3 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -202,16 +202,16 @@ then
 elif [ "$1" = "diag" ]
 then
     project="$2"
-    if [ -n "$project" ]
+    if [ -z "$project" ]
     then
         echo "You need to specify a project for diagnosis."
         exit 1
     fi
 
     destDir="$3"
-    if [ -n "$destDir" ]
+    if [ -z "$destDir" ]
     then
-        $destDir="$KYLIN_HOME/diagnosis_dump/"
+        destDir="$KYLIN_HOME/diagnosis_dump/"
         mkdir -p $destDir
     fi
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 4876dae..992fd5b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -120,10 +120,10 @@ public class KylinVersion {
     }
 
     public static void main(String[] args) {
-        System.out.println(getKylinDetailInformation());
+        System.out.println(getKylinClientInformation());
     }
 
-    public static String getKylinDetailInformation() {
+    public static String getKylinClientInformation() {
         StringBuilder buf = new StringBuilder();
 
         buf.append("kylin.version:").append(KylinVersion.getCurrentVersion()).append("\n");

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 01b05da..87a866a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -167,7 +167,9 @@ public class CubeManager implements IRealizationProvider {
         DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(),true, col,
factTableValueProvider);
 
         if (dictInfo != null) {
+            Dictionary dict = dictInfo.getDictionaryObject();
             cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
+            cubeSeg.getRowkeyStats().add(new Object[]{col.getName(), dict.getSize(), dict.getSizeOfId()});
 
             CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
             cubeBuilder.setToUpdateSegs(cubeSeg);

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 10642c4..0d8da6f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -94,6 +94,9 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment
     @JsonProperty("index_path")
     private String indexPath;
 
+    @JsonProperty("rowkey_stats")
+    private List<Object[]> rowkeyStats = Lists.newArrayList();
+
     private volatile Map<Long, Short> cuboidBaseShards = Maps.newHashMap();//cuboid
id ==> base(starting) shard for this cuboid
 
     public CubeDesc getCubeDesc() {
@@ -231,10 +234,13 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment
 
     @Override
     public String getStorageLocationIdentifier() {
-
         return storageLocationIdentifier;
     }
 
+    public List<Object[]> getRowkeyStats() {
+        return rowkeyStats;
+    }
+
     public Map<String, String> getDictionaries() {
         if (dictionaries == null)
             dictionaries = new ConcurrentHashMap<String, String>();

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 661e8e4..e2f4f3a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.hbase;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.lang.StringUtils;
@@ -46,6 +47,8 @@ import org.slf4j.LoggerFactory;
  */
 public class HBaseConnection {
 
+    public static final String HTABLE_UUID_TAG = "UUID";
+
     private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
 
     private static final Map<String, Configuration> ConfigCache = new ConcurrentHashMap<String,
Configuration>();
@@ -197,6 +200,8 @@ public class HBaseConnection {
                     desc.addFamily(fd);
                 }
             }
+
+            desc.setValue(HTABLE_UUID_TAG, UUID.randomUUID().toString());
             hbase.createTable(desc);
 
             logger.debug("HTable '" + tableName + "' created");

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 7df3f0c..7fc08f4 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -31,11 +31,6 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-source-hive</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kylin</groupId>
             <artifactId>kylin-source-kafka</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
@@ -49,16 +44,6 @@
             <artifactId>kylin-engine-mr</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-engine-streaming</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-invertedindex</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
 
         <!--Env-->
         <dependency>
@@ -73,4 +58,45 @@
         </dependency>
     </dependencies>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.3</version>
+
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <minimizeJar>false</minimizeJar>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <shadedClassifierName>tool</shadedClassifierName>
+                            <artifactSet>
+                                <excludes>
+                                    <exclude>io.netty:*</exclude>
+                                    <exclude>org.apache.zookeeper:*</exclude>
+                                    <exclude>net.sf.ehcache:*</exclude>
+                                </excludes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
 </project>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index e74b4eb..6b56201 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -39,7 +39,6 @@ import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.streaming.StreamingConfig;
 import org.apache.kylin.engine.streaming.StreamingManager;
-import org.apache.kylin.invertedindex.IIInstance;
 import org.apache.kylin.job.dao.ExecutableDao;
 import org.apache.kylin.job.dao.ExecutablePO;
 import org.apache.kylin.job.exception.PersistentException;
@@ -321,8 +320,6 @@ public class CubeMetaExtractor extends AbstractApplication {
                 }
                 retrieveResourcePath(iRealization);
             }
-        } else if (realization instanceof IIInstance) {
-            logger.warn("Does not support extract II instance or hybrid that contains II");
         } else {
             logger.warn("Unknown realization type: " + realization.getType());
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 98302ae..a403ee2 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -19,10 +19,9 @@
 package org.apache.kylin.tool;
 
 import java.io.File;
-import java.io.FilenameFilter;
+import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Calendar;
 import java.util.Date;
 
 import org.apache.commons.cli.Option;
@@ -30,10 +29,17 @@ import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinVersion;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.ZipFileUtils;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -140,7 +146,7 @@ public class DiagnosisInfoCLI extends AbstractApplication {
             }
         }
 
-        // export os conf - linux
+        // export os (linux)
         if (includeLinux) {
             File linuxDir = new File(exportDir, "linux");
             FileUtils.forceMkdir(linuxDir);
@@ -178,6 +184,11 @@ public class DiagnosisInfoCLI extends AbstractApplication {
             FileUtils.writeStringToFile(new File(basicDir, "process"), output);
             output = kylinConfig.getCliCommandExecutor().execute("lsb_release -a").getSecond();
             FileUtils.writeStringToFile(new File(basicDir, "lsb_release"), output);
+            output = KylinVersion.getKylinClientInformation();
+            FileUtils.writeStringToFile(new File(basicDir, "client"), output);
+            output = getHBaseMetaStoreId();
+            FileUtils.writeStringToFile(new File(basicDir, "client"), output, true);
+
         } catch (Exception e) {
             logger.warn("Error in export process info.", e);
         }
@@ -240,4 +251,11 @@ public class DiagnosisInfoCLI extends AbstractApplication {
         }
         return null;
     }
+
+    private String getHBaseMetaStoreId() throws IOException {
+        HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()));
+        String metaStoreName = kylinConfig.getMetadataUrlPrefix();
+        HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName));
+        return "MetaStore UUID: " + desc.getValue(HBaseConnection.HTABLE_UUID_TAG);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
index 0c1bf47..8d69805 100644
--- a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
@@ -30,8 +30,6 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.kylin.common.KylinConfig;
@@ -46,7 +44,6 @@ import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.project.RealizationEntry;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
index 43758e0..11abc38 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
@@ -32,8 +32,6 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.ResourceTool;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.common.ShellExecutable;
 import org.apache.kylin.job.constant.ExecutableConstants;
 import org.apache.kylin.job.dao.ExecutableDao;
 import org.apache.kylin.job.dao.ExecutablePO;
@@ -42,9 +40,6 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
-/**
- * Created by dongli on 3/29/16.
- */
 public class JobInfoExtractor extends AbstractApplication {
     private static final Logger logger = LoggerFactory.getLogger(JobInfoExtractor.class);
 
@@ -116,7 +111,7 @@ public class JobInfoExtractor extends AbstractApplication {
         executeExtraction(dest);
 
         if (includeCube) {
-            String cubeName = CubingExecutableUtil.getCubeName(executablePO.getParams());
+            String cubeName = executablePO.getParams().get("cubename");
             String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", dest + "cube_" + cubeName
+ "/", "-includeJobs", "false" };
             logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs));
             cubeMetaExtractor.execute(cubeMetaArgs);
@@ -125,7 +120,7 @@ public class JobInfoExtractor extends AbstractApplication {
         if (includeYarnLogs) {
             logger.info("Start to related yarn job logs: " + jobId);
             for (String taskId : yarnLogsResources) {
-                extractYarnLog(taskId, dest + "yarn_" + jobId + "/");
+                extractYarnLog(taskId, dest + "yarn_" + jobId + "/", true);
             }
         }
 
@@ -151,19 +146,30 @@ public class JobInfoExtractor extends AbstractApplication {
         }
     }
 
-    private void extractYarnLog(String taskId, String dest) throws Exception {
+    private void extractYarnLog(String taskId, String dest, boolean onlySucc) throws Exception
{
         final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo();
         if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) {
             String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job",
"application");
-            File destFile = new File(dest + applicationId + ".log");
+            if (!onlySucc || isYarnAppSucc(applicationId)) {
+                File destFile = new File(dest + applicationId + ".log");
 
-            ShellExecutable yarnExec = new ShellExecutable();
-            yarnExec.setCmd("yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath());
-            yarnExec.setName(yarnExec.getCmd());
+                String yarnCmd = "yarn logs -applicationId " + applicationId + " > " +
destFile.getAbsolutePath();
+                logger.info(yarnCmd);
+                kylinConfig.getCliCommandExecutor().execute(yarnCmd);
+            }
+        }
+    }
 
-            logger.info(yarnExec.getCmd());
-            kylinConfig.getCliCommandExecutor().execute(yarnExec.getCmd(), null);
+    private boolean isYarnAppSucc(String applicationId) throws IOException {
+        final String yarnCmd = "yarn application -status " + applicationId;
+        final String cmdOutput = kylinConfig.getCliCommandExecutor().execute(yarnCmd).getSecond();
+        final String[] cmdOutputLines = cmdOutput.split("\n");
+        for (String cmdOutputLine : cmdOutputLines) {
+            if (cmdOutputLine.equals("Final-State : SUCCEEDED")) {
+                return true;
+            }
         }
+        return false;
     }
 
     private void addRequired(String record) {


Mime
View raw message