carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [01/14] incubator-carbondata git commit: Fix for spark version 1.6.2 test cases
Date Thu, 16 Mar 2017 09:28:44 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 6e333f02a -> 892cf689c


Fix for spark version 1.6.2 test cases


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/06efc20f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/06efc20f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/06efc20f

Branch: refs/heads/master
Commit: 06efc20f9851cbecaff46b553c8bd10f53316e17
Parents: c230b8c
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Thu Mar 16 10:22:49 2017 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Thu Mar 16 14:50:43 2017 +0530

----------------------------------------------------------------------
 .../core/datastore/block/SegmentProperties.java |  26 ++-
 .../core/metadata/schema/table/CarbonTable.java | 158 ++++++-------------
 .../csvbased/CarbonCSVBasedSeqGenMeta.java      |  14 +-
 3 files changed, 82 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/06efc20f/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 84bf208..6c7d014 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -326,7 +326,7 @@ public class SegmentProperties {
     // to store the ordinal of the column group ordinal
     int columnGroupOrdinal = 0;
     int counter = 0;
-    int complexTypeOrdinal = 0;
+    int complexTypeOrdinal = -1;
     while (counter < columnsInTable.size()) {
       columnSchema = columnsInTable.get(counter);
       if (columnSchema.isDimensionColumn()) {
@@ -366,7 +366,7 @@ public class SegmentProperties {
             new DataType[] { DataType.ARRAY, DataType.STRUCT })) {
           cardinalityIndexForComplexDimensionColumn.add(tableOrdinal);
           carbonDimension =
-              new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, complexTypeOrdinal++);
+              new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, ++complexTypeOrdinal);
           carbonDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
           complexDimensions.add(carbonDimension);
           isComplexDimensionStarted = true;
@@ -379,9 +379,7 @@ public class SegmentProperties {
             cardinalityIndexForComplexDimensionColumn.add(++tableOrdinal);
           }
           counter = dimensonOrdinal;
-          complexTypeOrdinal = carbonDimension.getListOfChildDimensions()
-              .get(carbonDimension.getListOfChildDimensions().size() - 1).getComplexTypeOrdinal();
-          complexTypeOrdinal++;
+          complexTypeOrdinal = assignComplexOrdinal(carbonDimension, complexTypeOrdinal);
           continue;
         } else {
           // for no dictionary dimension
@@ -446,6 +444,24 @@ public class SegmentProperties {
   }
 
   /**
+   * Read all primitive/complex children and set it as list of child carbon dimension to
parent
+   * dimension
+   */
+  private int assignComplexOrdinal(CarbonDimension parentDimension, int complexDimensionOrdianl)
{
+    for (int i = 0; i < parentDimension.getNumberOfChild(); i++) {
+      CarbonDimension dimension = parentDimension.getListOfChildDimensions().get(i);
+      if (dimension.getNumberOfChild() > 0) {
+        dimension.setComplexTypeOridnal(++complexDimensionOrdianl);
+        complexDimensionOrdianl = assignComplexOrdinal(dimension, complexDimensionOrdianl);
+      } else {
+        parentDimension.getListOfChildDimensions().get(i)
+            .setComplexTypeOridnal(++complexDimensionOrdianl);
+      }
+    }
+    return complexDimensionOrdianl;
+  }
+
+  /**
    * Below method will fill the key generator detail of both the type of key
    * generator. This will be required for during both query execution and data
    * loading.

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/06efc20f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 4527b77..dbc8836 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -55,46 +55,36 @@ public class CarbonTable implements Serializable {
   private AbsoluteTableIdentifier absoluteTableIdentifier;
 
   /**
-   * TableName, Dimensions list. This map will contain dimensions which are visible
+   * TableName, Dimensions list. This map will contain allDimensions which are visible
    */
-  private transient Map<String, List<CarbonDimension>> tableDimensionsMap;
+  private Map<String, List<CarbonDimension>> tableDimensionsMap;
 
   /**
-   * TableName, Dimensions list. This map will contain both visible and not visible dimensions
+   * list of all the allDimensions
    */
-  private transient Map<String, List<CarbonDimension>> tableDimensionsMapWithDeletedColumns;
-
-  /**
-   * list of all the dimensions
-   */
-  private List<CarbonDimension> dimensions;
+  private List<CarbonDimension> allDimensions;
 
   private Map<String, List<CarbonColumn>> createOrderColumn;
 
   /**
-   * TableName, Dimensions and children dimensions list
+   * TableName, Dimensions and children allDimensions list
    */
   private Map<String, List<CarbonDimension>> tablePrimitiveDimensionsMap;
 
   /**
-   * table measures list.
+   * table allMeasures list.
    */
   private Map<String, List<CarbonDimension>> tableImplicitDimensionsMap;
 
   /**
-   * table measures list. This map will contain dimensions which are visible
-   */
-  private transient Map<String, List<CarbonMeasure>> tableMeasuresMap;
-
-  /**
-   * table measures list. This map will contain both visible and not visible measures
+   * table allMeasures list. This map will contain allDimensions which are visible
    */
-  private transient Map<String, List<CarbonMeasure>> tableMeasuresMapWithDeletedMeasures;
+  private Map<String, List<CarbonMeasure>> tableMeasuresMap;
 
   /**
-   * list of measures
+   * list of allMeasures
    */
-  private List<CarbonMeasure> measures;
+  private List<CarbonMeasure> allMeasures;
 
   /**
    * table bucket map.
@@ -127,7 +117,9 @@ public class CarbonTable implements Serializable {
   private int blockSize;
 
   public CarbonTable() {
+    this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
     this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
+    this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
     this.tableBucketMap = new HashMap<>();
     this.aggregateTablesName = new ArrayList<String>();
     this.createOrderColumn = new HashMap<String, List<CarbonColumn>>();
@@ -167,8 +159,6 @@ public class CarbonTable implements Serializable {
    */
   private void fillCreateOrderColumn(String tableName) {
     List<CarbonColumn> columns = new ArrayList<CarbonColumn>();
-    fillVisibleDimensions(tableName);
-    fillVisibleMeasures(tableName);
     List<CarbonDimension> dimensions = this.tableDimensionsMap.get(tableName);
     List<CarbonMeasure> measures = this.tableMeasuresMap.get(tableName);
     Iterator<CarbonDimension> dimItr = dimensions.iterator();
@@ -215,15 +205,15 @@ public class CarbonTable implements Serializable {
   }
 
   /**
-   * Fill dimensions and measures for carbon table
+   * Fill allDimensions and allMeasures for carbon table
    *
    * @param tableSchema
    */
   private void fillDimensionsAndMeasuresForTables(TableSchema tableSchema) {
     List<CarbonDimension> primitiveDimensions = new ArrayList<CarbonDimension>();
     List<CarbonDimension> implicitDimensions = new ArrayList<CarbonDimension>();
-    dimensions = new ArrayList<>();
-    measures = new ArrayList<>();
+    allDimensions = new ArrayList<CarbonDimension>();
+    allMeasures = new ArrayList<CarbonMeasure>();
     this.tablePrimitiveDimensionsMap.put(this.tableUniqueName, primitiveDimensions);
     this.tableImplicitDimensionsMap.put(tableSchema.getTableName(), implicitDimensions);
     int dimensionOrdinal = 0;
@@ -241,7 +231,7 @@ public class CarbonTable implements Serializable {
               new CarbonDimension(columnSchema, dimensionOrdinal++,
                     columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal);
           complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
-          dimensions.add(complexDimension);
+          allDimensions.add(complexDimension);
           dimensionOrdinal =
               readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
                   listOfColumns, complexDimension, primitiveDimensions);
@@ -252,14 +242,14 @@ public class CarbonTable implements Serializable {
             CarbonDimension dimension =
                     new CarbonDimension(columnSchema, dimensionOrdinal++,
                             columnSchema.getSchemaOrdinal(), -1, -1, -1);
-            dimensions.add(dimension);
+            allDimensions.add(dimension);
             primitiveDimensions.add(dimension);
           } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
               && columnSchema.getColumnGroupId() == -1) {
             CarbonDimension dimension =
                     new CarbonDimension(columnSchema, dimensionOrdinal++,
                             columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1);
-            dimensions.add(dimension);
+            allDimensions.add(dimension);
             primitiveDimensions.add(dimension);
           } else {
             columnGroupOrdinal =
@@ -268,15 +258,17 @@ public class CarbonTable implements Serializable {
             CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++,
                     columnSchema.getSchemaOrdinal(), keyOrdinal++,
                     columnGroupOrdinal, -1);
-            dimensions.add(dimension);
+            allDimensions.add(dimension);
             primitiveDimensions.add(dimension);
           }
         }
       } else {
-        measures.add(new CarbonMeasure(columnSchema, measureOrdinal++,
+        allMeasures.add(new CarbonMeasure(columnSchema, measureOrdinal++,
                  columnSchema.getSchemaOrdinal()));
       }
     }
+    fillVisibleDimensions(tableSchema.getTableName());
+    fillVisibleMeasures(tableSchema.getTableName());
     addImplicitDimension(dimensionOrdinal, implicitDimensions);
   }
 
@@ -404,18 +396,16 @@ public class CarbonTable implements Serializable {
    * @return number of dimension present the table
    */
   public int getNumberOfDimensions(String tableName) {
-    fillVisibleDimensions(tableName);
     return tableDimensionsMap.get(tableName).size();
   }
 
   /**
-   * to get the number of measures present in the table
+   * to get the number of allMeasures present in the table
    *
    * @param tableName
-   * @return number of measures present the table
+   * @return number of allMeasures present the table
    */
   public int getNumberOfMeasures(String tableName) {
-    fillVisibleMeasures(tableName);
     return tableMeasuresMap.get(tableName).size();
   }
 
@@ -426,7 +416,6 @@ public class CarbonTable implements Serializable {
    * @return all dimension of a table
    */
   public List<CarbonDimension> getDimensionByTableName(String tableName) {
-    fillVisibleDimensions(tableName);
     return tableDimensionsMap.get(tableName);
   }
 
@@ -437,7 +426,6 @@ public class CarbonTable implements Serializable {
    * @return all measure of a table
    */
   public List<CarbonMeasure> getMeasureByTableName(String tableName) {
-    fillVisibleMeasures(tableName);
     return tableMeasuresMap.get(tableName);
   }
 
@@ -468,10 +456,9 @@ public class CarbonTable implements Serializable {
    * @return
    */
   public CarbonMeasure getMeasureByName(String tableName, String columnName) {
-    fillVisibleMeasures(tableName);
     List<CarbonMeasure> measureList = tableMeasuresMap.get(tableName);
     for (CarbonMeasure measure : measureList) {
-      if (!measure.isInvisible() && measure.getColName().equalsIgnoreCase(columnName))
{
+      if (measure.getColName().equalsIgnoreCase(columnName)) {
         return measure;
       }
     }
@@ -487,10 +474,9 @@ public class CarbonTable implements Serializable {
    */
   public CarbonDimension getDimensionByName(String tableName, String columnName) {
     CarbonDimension carbonDimension = null;
-    fillVisibleDimensions(tableName);
     List<CarbonDimension> dimList = tableDimensionsMap.get(tableName);
     for (CarbonDimension dim : dimList) {
-      if (!dim.isInvisible() && dim.getColName().equalsIgnoreCase(columnName)) {
+      if (dim.getColName().equalsIgnoreCase(columnName)) {
         carbonDimension = dim;
         break;
       }
@@ -515,7 +501,7 @@ public class CarbonTable implements Serializable {
     Iterator<CarbonColumn> colItr = columns.iterator();
     while (colItr.hasNext()) {
       CarbonColumn col = colItr.next();
-      if (!col.isInvisible() && col.getColName().equalsIgnoreCase(columnName)) {
+      if (col.getColName().equalsIgnoreCase(columnName)) {
         return col;
       }
     }
@@ -525,10 +511,9 @@ public class CarbonTable implements Serializable {
    * gets all children dimension for complex type
    *
    * @param dimName
-   * @return list of child dimensions
+   * @return list of child allDimensions
    */
   public List<CarbonDimension> getChildren(String dimName) {
-    fillVisibleDimensions(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
     for (List<CarbonDimension> list : tableDimensionsMap.values()) {
       List<CarbonDimension> childDims = getChildren(dimName, list);
       if (childDims != null) {
@@ -539,15 +524,15 @@ public class CarbonTable implements Serializable {
   }
 
   /**
-   * returns level 2 or more child dimensions
+   * returns level 2 or more child allDimensions
    *
    * @param dimName
    * @param dimensions
-   * @return list of child dimensions
+   * @return list of child allDimensions
    */
   public List<CarbonDimension> getChildren(String dimName, List<CarbonDimension>
dimensions) {
     for (CarbonDimension carbonDimension : dimensions) {
-      if (!carbonDimension.isInvisible() && carbonDimension.getColName().equals(dimName))
{
+      if (carbonDimension.getColName().equals(dimName)) {
         return carbonDimension.getListOfChildDimensions();
       } else if (null != carbonDimension.getListOfChildDimensions()
           && carbonDimension.getListOfChildDimensions().size() > 0) {
@@ -608,95 +593,50 @@ public class CarbonTable implements Serializable {
   }
 
   /**
-   * fill all dimensions if empty and return the map
+   * return all allDimensions in the table
    *
-   * @param tableName
    * @return
    */
-  public Map<String, List<CarbonDimension>> getTableDimensionsMapWithDeletedColumns(
-      String tableName) {
-    if (null == tableDimensionsMapWithDeletedColumns || null == tableDimensionsMapWithDeletedColumns
-        .get(tableName)) {
-      synchronized (this) {
-        if (null == tableDimensionsMapWithDeletedColumns
-            || null == tableDimensionsMapWithDeletedColumns.get(tableName)) {
-          if (null == tableDimensionsMapWithDeletedColumns) {
-            tableDimensionsMapWithDeletedColumns = new HashMap<>();
-          }
-          tableDimensionsMapWithDeletedColumns.put(tableName, dimensions);
-        }
-      }
-    }
-    return tableDimensionsMapWithDeletedColumns;
+  public List<CarbonDimension> getAllDimensions() {
+    return allDimensions;
   }
 
   /**
-   * This method will all the visible dimensions
+   * This method will all the visible allDimensions
    *
    * @param tableName
    */
   private void fillVisibleDimensions(String tableName) {
-    if (null == tableDimensionsMap || null == tableDimensionsMap.get(tableName)) {
-      synchronized (this) {
-        if (null == tableDimensionsMap || null == tableDimensionsMap.get(tableName)) {
-          if (null == tableDimensionsMap) {
-            tableDimensionsMap = new HashMap<>();
-          }
-          List<CarbonDimension> visibleDimensions = new ArrayList<>(dimensions.size());
-          for (CarbonDimension dimension : dimensions) {
-            if (!dimension.isInvisible()) {
-              visibleDimensions.add(dimension);
-            }
-          }
-          tableDimensionsMap.put(tableName, visibleDimensions);
-        }
+    List<CarbonDimension> visibleDimensions = new ArrayList<CarbonDimension>(allDimensions.size());
+    for (CarbonDimension dimension : allDimensions) {
+      if (!dimension.isInvisible()) {
+        visibleDimensions.add(dimension);
       }
     }
+    tableDimensionsMap.put(tableName, visibleDimensions);
   }
 
   /**
-   * fill all measures if empty and return the map
+   * return all allMeasures in the table
    *
-   * @param tableName
    * @return
    */
-  public Map<String, List<CarbonMeasure>> getTableMeasuresMapWithDeletedMeasures(String
tableName) {
-    if (null == tableMeasuresMapWithDeletedMeasures || null == tableMeasuresMapWithDeletedMeasures
-        .get(tableName)) {
-      synchronized (this) {
-        if (null == tableMeasuresMapWithDeletedMeasures
-            || null == tableMeasuresMapWithDeletedMeasures.get(tableName)) {
-          if (null == tableMeasuresMapWithDeletedMeasures) {
-            tableMeasuresMapWithDeletedMeasures = new HashMap<>();
-          }
-          tableMeasuresMapWithDeletedMeasures.put(tableName, measures);
-        }
-      }
-    }
-    return tableMeasuresMapWithDeletedMeasures;
+  public List<CarbonMeasure> getAllMeasures() {
+    return allMeasures;
   }
 
   /**
-   * This method will all the visible measures
+   * This method will all the visible allMeasures
    *
    * @param tableName
    */
   private void fillVisibleMeasures(String tableName) {
-    if (null == tableMeasuresMap || null == tableMeasuresMap.get(tableName)) {
-      synchronized (this) {
-        if (null == tableMeasuresMap || null == tableMeasuresMap.get(tableName)) {
-          if (null == tableMeasuresMap) {
-            tableMeasuresMap = new HashMap<>();
-          }
-          List<CarbonMeasure> visibleMeasures = new ArrayList<>(measures.size());
-          for (CarbonMeasure measure : measures) {
-            if (!measure.isInvisible()) {
-              visibleMeasures.add(measure);
-            }
-          }
-          tableMeasuresMap.put(tableName, visibleMeasures);
-        }
+    List<CarbonMeasure> visibleMeasures = new ArrayList<CarbonMeasure>(allMeasures.size());
+    for (CarbonMeasure measure : allMeasures) {
+      if (!measure.isInvisible()) {
+        visibleMeasures.add(measure);
       }
     }
+    tableMeasuresMap.put(tableName, visibleMeasures);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/06efc20f/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
index 92784ba..5419858 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
@@ -777,7 +777,12 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
   }
 
   private void updateMeasureAggregator(String msrAggregatorString) {
-    String[] split = msrAggregatorString.split(CarbonCommonConstants.SEMICOLON_SPC_CHARACTER);
+    String[] split = null;
+    if (msrAggregatorString == null) {
+      split =  new String[0];
+    } else {
+      split = msrAggregatorString.split(CarbonCommonConstants.SEMICOLON_SPC_CHARACTER);
+    }
     msrAggregators = new String[split.length];
     System.arraycopy(split, 0, msrAggregators, 0, split.length);
   }
@@ -1070,7 +1075,12 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements
StepMetaIn
     // get high cardinality dimension Array
     noDictionaryCols = RemoveDictionaryUtil.extractNoDictionaryDimsArr(noDictionaryDims);
 
-    String[] sm = msr.split(CarbonCommonConstants.COMA_SPC_CHARACTER);
+    String[] sm = null;
+    if (null != msr) {
+      sm = msr.split(CarbonCommonConstants.COMA_SPC_CHARACTER);
+    } else {
+      sm = new String[0];
+    }
     int[] m = new int[sm.length];
     Set<String> mlist = new LinkedHashSet<String>();
     for (int i = 0; i < m.length; i++) {


Mime
View raw message