kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [11/50] incubator-kylin git commit: Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Date Fri, 03 Apr 2015 13:03:26 GMT
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict

Conflicts:
	storage/src/main/java/org/apache/kylin/storage/cube/CubeGridTable.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5612b126
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5612b126
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5612b126

Branch: refs/heads/streaming-localdict
Commit: 5612b126c60c6c0b67e5848654fbd577010411ba
Parents: c13ecb2 95a3e79
Author: liyang@apache.org <yangli9@D-SHC-00801746.corp.ebay.com>
Authored: Thu Apr 2 11:08:56 2015 +0100
Committer: liyang@apache.org <yangli9@D-SHC-00801746.corp.ebay.com>
Committed: Thu Apr 2 11:08:56 2015 +0100

----------------------------------------------------------------------
 bin/kylin.sh                                    |  74 ++++--
 .../org/apache/kylin/common/KylinConfig.java    |  20 ++
 .../org/apache/kylin/common/util/BasicTest.java |   8 +-
 conf/kylin.properties                           |   5 +
 .../cube/estimation/CubeSizeEstimationCLI.java  |   2 +-
 .../kylin/dict/lookup/HiveTableReader.java      |   5 -
 .../apache/kylin/dict/HiveTableReaderTest.java  |   4 +-
 .../kylin/dict/lookup/LookupTableTest.java      |   6 +-
 .../test_case_data/sandbox/kylin.properties     |  10 +-
 .../apache/kylin/invertedindex/IIManager.java   |   5 +
 .../invertedindex/index/BatchSliceBuilder.java  |   1 -
 .../invertedindex/index/BitMapContainer.java    |  24 +-
 .../invertedindex/index/RawTableRecord.java     |   1 -
 .../invertedindex/IIDescManagerTest.java        |  10 +-
 .../invertedindex/invertedindex/IIDescTest.java |   1 -
 .../apache/kylin/job/CubeMetadataUpgrade.java   |  35 +--
 .../org/apache/kylin/job/JoinedFlatTable.java   |   2 +-
 .../kylin/job/constant/BatchConstants.java      |   3 +-
 .../kylin/job/constant/ExecutableConstants.java |   2 +-
 .../apache/kylin/job/cube/CubingJobBuilder.java |  37 ++-
 .../cardinality/HiveColumnCardinalityJob.java   |   2 -
 .../kylin/job/hadoop/cube/CubeHFileJob.java     |  13 +-
 .../kylin/job/hadoop/cube/CubeHFileMapper.java  |  93 +------
 .../job/hadoop/cube/FactDistinctColumnsJob.java |   5 +-
 .../hadoop/cube/FactDistinctColumnsReducer.java | 116 +++++---
 .../cube/FactDistinctHiveColumnsMapper.java     |   2 +-
 .../cube/FactDistinctIIColumnsMapper.java       |   1 -
 .../kylin/job/hadoop/cube/KeyValueCreator.java  | 104 ++++++++
 .../kylin/job/hadoop/cube/MergeCuboidJob.java   |   2 -
 .../hadoop/cube/RangeKeyDistributionJob.java    |   2 -
 .../hadoop/cubev2/BuildDictionaryMapper.java    | 184 -------------
 .../kylin/job/hadoop/cubev2/InMemCuboidJob.java |  98 ++-----
 .../job/hadoop/cubev2/InMemCuboidMapper.java    |  74 ++----
 .../job/hadoop/cubev2/InMemCuboidReducer.java   |  55 +++-
 .../kylin/job/hadoop/hbase/CreateHTableJob.java | 193 ++++++++++++--
 .../hadoop/invertedindex/IICreateHFileJob.java  |   4 +-
 .../kylin/job/streaming/StreamingBootstrap.java |  45 +++-
 .../kylin/job/streaming/StreamingCLI.java       |   6 +-
 .../kylin/job/tools/CubeMigrationCLI.java       |   1 -
 .../kylin/job/BuildCubeWithStreamTest.java      |  43 ++-
 .../apache/kylin/job/BuildIIWithStreamTest.java |   1 +
 .../job/hadoop/cube/MergeCuboidMapperTest.java  |   4 +-
 .../job/hadoop/invertedindex/II2CubeTest.java   |   1 -
 .../kylin/metadata/project/ProjectL2Cache.java  |   4 +-
 .../org/apache/kylin/jdbc/JDBCDriverTest.java   |   1 -
 .../rest/controller/CacheControllerTest.java    |   9 +-
 .../kylin/rest/service/CacheServiceTest.java    |   2 -
 .../kylin/storage/cube/CubeGridTable.java       |   8 +-
 .../hbase/coprocessor/CoprocessorRowType.java   |   1 -
 .../endpoint/EndpointTupleIterator.java         |   4 +-
 .../kylin/storage/hybrid/HybridManagerTest.java |  10 -
 .../minicluster/HiveMiniClusterTest.java        | 262 +++++++++----------
 .../apache/kylin/streaming/KafkaConsumer.java   |  17 ++
 .../apache/kylin/streaming/KafkaRequester.java  |   2 +-
 .../apache/kylin/streaming/StreamBuilder.java   |  15 +-
 .../kylin/streaming/StringStreamParser.java     |   6 +-
 .../kylin/streaming/cube/CubeStreamBuilder.java |  44 +---
 .../invertedindex/IIStreamBuilder.java          |  42 +--
 .../streaming/invertedindex/SliceBuilder.java   |   4 +-
 59 files changed, 890 insertions(+), 845 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5612b126/storage/src/main/java/org/apache/kylin/storage/cube/CubeGridTable.java
----------------------------------------------------------------------
diff --cc storage/src/main/java/org/apache/kylin/storage/cube/CubeGridTable.java
index d2b0976,7923631..641a269
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeGridTable.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeGridTable.java
@@@ -20,33 -18,10 +20,33 @@@ import com.google.common.collect.Maps
  public class CubeGridTable {
  
      @SuppressWarnings("rawtypes")
-     public static Map<TblColRef, Dictionary> getDimensionToDictionaryMap(CubeSegment
cubeSeg, long cuboidId) {
++    public static Map<TblColRef, Dictionary<?>> getDimensionToDictionaryMap(CubeSegment
cubeSeg, long cuboidId) {
 +        CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 +        CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
 +
 +        // build a dictionary map
-         Map<TblColRef, Dictionary> dictionaryMap = Maps.newHashMap();
++        Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
 +        List<TblColRef> dimCols = Cuboid.findById(cubeDesc, cuboidId).getColumns();
 +        for (TblColRef col : dimCols) {
 +            Dictionary<?> dictionary = cubeMgr.getDictionary(cubeSeg, col);
 +            if (dictionary != null) {
 +                dictionaryMap.put(col, dictionary);
 +            }
 +        }
 +        return dictionaryMap;
 +    }
 +
 +    @SuppressWarnings("rawtypes")
 +    public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
-         Map<TblColRef, Dictionary> dictionaryMap = getDimensionToDictionaryMap(cubeSeg,
cuboidId);
++        Map<TblColRef, Dictionary<?>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg,
cuboidId);
 +        return newGTInfo(cubeSeg.getCubeDesc(), cuboidId, dictionaryMap);
 +    }
 +    
 +    @SuppressWarnings("rawtypes")
-     public static GTInfo newGTInfo(CubeDesc cubeDesc, long cuboidId, Map<TblColRef, Dictionary>
dictionaryMap) {
+     public static GTInfo newGTInfo(CubeDesc cubeDesc, long cuboidId, Map<TblColRef, Dictionary<?>>
dictionaryMap) {
          Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
          List<TblColRef> dimCols = cuboid.getColumns();
 -        
 +
          int nColumns = dimCols.size() + cubeDesc.getMeasures().size();
          BitSet dimensions = new BitSet();
          dimensions.set(0, dimCols.size());


Mime
View raw message