kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billy...@apache.org
Subject kylin git commit: fix FileSystem.close [Forced Update!]
Date Tue, 03 Jan 2017 12:37:25 GMT
Repository: kylin
Updated Branches:
  refs/heads/fix-sonar-1 a7f0f49e8 -> 7972ac5bb (forced update)


fix FileSystem.close


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7972ac5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7972ac5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7972ac5b

Branch: refs/heads/fix-sonar-1
Commit: 7972ac5bb86aee57127eeafa6cee7ca3b5548cd9
Parents: 6766894
Author: Billy Liu <billyliu@apache.org>
Authored: Tue Jan 3 20:37:13 2017 +0800
Committer: Billy Liu <billyliu@apache.org>
Committed: Tue Jan 3 20:37:13 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/util/CliCommandExecutor.java | 15 +++++++++------
 .../org/apache/kylin/dict/AppendTrieDictionary.java  | 11 ++++++++++-
 .../java/org/apache/kylin/engine/mr/HadoopUtil.java  |  1 +
 .../kylin/engine/mr/steps/MergeStatisticsStep.java   |  1 +
 .../kylin/engine/mr/steps/SaveStatisticsStep.java    |  1 +
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java       |  1 +
 .../apache/kylin/engine/spark/SparkCountDemo.java    |  2 +-
 .../org/apache/kylin/source/hive/HiveMRInput.java    |  1 +
 .../cardinality/HiveColumnCardinalityUpdateJob.java  |  1 +
 .../org/apache/kylin/source/kafka/KafkaMRInput.java  |  1 +
 .../apache/kylin/storage/hbase/HBaseConnection.java  |  5 ++++-
 .../org/apache/kylin/tool/StorageCleanupJob.java     |  2 ++
 12 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index f97b609..488af5f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -131,18 +131,21 @@ public class CliCommandExecutor {
         BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
         String line;
         StringBuilder result = new StringBuilder();
-        while ((line = reader.readLine()) != null) {
-            result.append(line).append('\n');
-            if (logAppender != null) {
-                logAppender.log(line);
+        try {
+            while ((line = reader.readLine()) != null) {
+                result.append(line).append('\n');
+                if (logAppender != null) {
+                    logAppender.log(line);
+                }
             }
-        }
 
-        try {
             int exitCode = proc.waitFor();
             return Pair.newPair(exitCode, result.toString());
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new IOException(e);
+        } finally {
+            reader.close();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 32bfde6..1d17569 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -161,6 +161,15 @@ public class AppendTrieDictionary<T> extends Dictionary<T>
{
         }
 
         @Override
+        public boolean equals(Object obj) {
+            if (!(obj instanceof DictSliceKey)){
+                return false;
+            }
+            DictSliceKey other = (DictSliceKey)obj;
+            return Bytes.equals(key, other.key);
+        }
+
+        @Override
         public int compareTo(Object o) {
             if (!(o instanceof DictSliceKey)) {
                 return -1;
@@ -1219,7 +1228,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T>
{
             dstFs.delete(dstPath, true);
         }
         FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(),
conf), dstPath, false, true, conf);
-
+        dstFs.close();
         return newDict;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..55a73ad 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -106,6 +106,7 @@ public class HadoopUtil {
         if (fs.exists(path)) {
             fs.delete(path, true);
         }
+        fs.close();
     }
 
     public static byte[] toBytes(Writable writable) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 811fc24..8227b68 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -115,6 +115,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
                     e.printStackTrace();
                     throw e;
                 } finally {
+                    fs.close();
                     IOUtils.closeStream(reader);
                     if (tempFile != null)
                         tempFile.delete();

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 020c62c..1a4e502 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -70,6 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
                 String statisticsFileName = newSegment.getStatisticsResourcePath();
                 rs.putResource(statisticsFileName, is, System.currentTimeMillis());
             } finally {
+                fs.close();
                 IOUtils.closeStream(is);
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dcc9190..dbeec53 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -91,6 +91,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
             minValue = Math.min(min, minValue);
             maxValue = Math.max(max, maxValue);
         } finally {
+            fs.close();
             IOUtils.closeQuietly(is);
         }
         logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue);

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
index 6478c10..8a1a42e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
@@ -75,6 +75,6 @@ public class SparkCountDemo extends AbstractApplication {
                 return new Tuple2(key, value);
             }
         }).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class,
KeyValue.class, HFileOutputFormat.class);
-
+        sc.close();
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..7c7a5d1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -383,6 +383,7 @@ public class HiveMRInput implements IMRInput {
             if (fs.exists(externalDataPath)) {
                 fs.delete(externalDataPath, true);
             }
+            fs.close();
         }
 
         public void setIntermediateTableIdentity(String tableIdentity) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 246822c..b52f204 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -155,6 +155,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob
{
                 results.add(str);
             }
         }
+        fileSystem.close();
         return results;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index dde3584..62038db 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -227,6 +227,7 @@ public class KafkaMRInput implements IMRInput {
             if (fs.exists(externalDataPath)) {
                 fs.delete(externalDataPath, true);
             }
+            fs.close();
         }
 
         public void setDataPath(String externalDataPath) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..972d637 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -124,6 +124,7 @@ public class HBaseConnection {
                 coprocessorPool.shutdownNow();
             }
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             coprocessorPool.shutdownNow();
         }
     }
@@ -205,7 +206,9 @@ public class HBaseConnection {
     public static String makeQualifiedPathInHBaseCluster(String path) {
         try {
             FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-            return fs.makeQualified(new Path(path)).toString();
+            String result =  fs.makeQualified(new Path(path)).toString();
+            fs.close();
+            return result;
         } catch (IOException e) {
             throw new IllegalArgumentException("Cannot create FileSystem from current hbase
cluster conf", e);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 05e0142..5b993fb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -243,6 +243,7 @@ public class StorageCleanupJob extends AbstractApplication {
             }
             System.out.println("-------------------------------------------------------");
         }
+        fs.close();
     }
 
     private void cleanUnusedIntermediateHiveTable(Configuration conf) throws Exception {
@@ -349,6 +350,7 @@ public class StorageCleanupJob extends AbstractApplication {
                         } else {
                             logger.info("Hive table {}'s external path {} not exist. It's
normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path);
                         }
+                        fs.close();
                     } else {
                         logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}",
tableToDelete, segmentId2JobId.toString());
                     }


Mime
View raw message