carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/4] carbondata git commit: add TablePage
Date Sat, 27 May 2017 12:50:19 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master ffaeb80f2 -> dc0f56809


http://git-wip-us.apache.org/repos/asf/carbondata/blob/353272ef/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
index d6f992c..7c2fab4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
@@ -18,104 +18,13 @@
 package org.apache.carbondata.processing.util;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.IgnoreDictionary;
+import org.apache.carbondata.processing.newflow.row.WriteStepRowUtil;
 
 /**
  * This is the utility class for No Dictionary changes.
  */
 public class NonDictionaryUtil {
-  /**
-   * Here we are dividing one single object [] into 3 arrays. one for
-   * dimensions , one for high card, one for measures.
-   *
-   * @param out
-   * @param byteBufferArr
-   */
-  public static void prepareOut(Object[] newOutArr, ByteBuffer[] byteBufferArr, Object[]
out,
-      int dimCount) {
-
-    byte[] nonDictionaryCols =
-        NonDictionaryUtil.packByteBufferIntoSingleByteArray(byteBufferArr);
-    Integer[] dimArray = new Integer[dimCount];
-    for (int i = 0; i < dimCount; i++) {
-      dimArray[i] = (Integer) out[i];
-    }
-
-    Object[] measureArray = new Object[out.length - dimCount];
-    int index = 0;
-    for (int j = dimCount; j < out.length; j++) {
-      measureArray[index++] = out[j];
-    }
-
-    newOutArr[IgnoreDictionary.DIMENSION_INDEX_IN_ROW.getIndex()] = dimArray;
-    newOutArr[IgnoreDictionary.BYTE_ARRAY_INDEX_IN_ROW.getIndex()] = nonDictionaryCols;
-    newOutArr[IgnoreDictionary.MEASURES_INDEX_IN_ROW.getIndex()] = measureArray;
-
-  }
-
-  /**
-   * This method will form one single byte [] for all the high card dims.
-   *
-   * @param byteBufferArr
-   * @return
-   */
-  public static byte[] packByteBufferIntoSingleByteArray(ByteBuffer[] byteBufferArr) {
-    // for empty array means there is no data to remove dictionary.
-    if (null == byteBufferArr || byteBufferArr.length == 0) {
-      return null;
-    }
-    int noOfCol = byteBufferArr.length;
-    short toDetermineLengthOfByteArr = 2;
-    short offsetLen = (short) (noOfCol * 2 + toDetermineLengthOfByteArr);
-    int totalBytes = calculateTotalBytes(byteBufferArr) + offsetLen;
-
-    ByteBuffer buffer = ByteBuffer.allocate(totalBytes);
-
-    // write the length of the byte [] as first short
-    buffer.putShort((short) (totalBytes - toDetermineLengthOfByteArr));
-    // writing the offset of the first element.
-    buffer.putShort(offsetLen);
-
-    // prepare index for byte []
-    for (int index = 0; index < byteBufferArr.length - 1; index++) {
-      ByteBuffer individualCol = byteBufferArr[index];
-      int noOfBytes = individualCol.capacity();
-
-      buffer.putShort((short) (offsetLen + noOfBytes));
-      offsetLen += noOfBytes;
-      individualCol.rewind();
-    }
-
-    // put actual data.
-    for (int index = 0; index < byteBufferArr.length; index++) {
-      ByteBuffer individualCol = byteBufferArr[index];
-      buffer.put(individualCol.array());
-    }
-
-    buffer.rewind();
-    return buffer.array();
-
-  }
-
-  /**
-   * To calculate the total bytes in byte Buffer[].
-   *
-   * @param byteBufferArr
-   * @return
-   */
-  private static int calculateTotalBytes(ByteBuffer[] byteBufferArr) {
-    int total = 0;
-    for (int index = 0; index < byteBufferArr.length; index++) {
-      total += byteBufferArr[index].capacity();
-    }
-    return total;
-  }
 
   /**
    * This method will form one single byte [] for all the high card dims.
@@ -173,51 +82,6 @@ public class NonDictionaryUtil {
   }
 
   /**
-   * Method to check whether entire row is empty or not.
-   *
-   * @param row
-   * @return
-   */
-  public static boolean checkAllValuesForNull(Object[] row) {
-    if (checkNullForDims(row[0]) && checkNullForMeasures(row[2]) && null
== row[1]) {
-      return true;
-    }
-    return false;
-  }
-
-  /**
-   * To check whether the measures are empty/null
-   *
-   * @param object
-   * @return
-   */
-  private static boolean checkNullForMeasures(Object object) {
-    Object[] measures = (Object[]) object;
-    for (Object measure : measures) {
-      if (null != measure) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  /**
-   * To check whether the dimensions are empty/null
-   *
-   * @param object
-   * @return
-   */
-  private static boolean checkNullForDims(Object object) {
-    Integer[] dimensions = (Integer[]) object;
-    for (Integer dimension : dimensions) {
-      if (null != dimension) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  /**
    * Method to get the required Dimension from obj []
    *
    * @param index
@@ -226,7 +90,7 @@ public class NonDictionaryUtil {
    */
   public static Integer getDimension(int index, Object[] row) {
 
-    Integer[] dimensions = (Integer[]) row[IgnoreDictionary.DIMENSION_INDEX_IN_ROW.getIndex()];
+    Integer[] dimensions = (Integer[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
 
     return dimensions[index];
 
@@ -240,194 +104,22 @@ public class NonDictionaryUtil {
    * @return
    */
   public static Object getMeasure(int index, Object[] row) {
-    Object[] measures = (Object[]) row[IgnoreDictionary.MEASURES_INDEX_IN_ROW.getIndex()];
+    Object[] measures = (Object[]) row[WriteStepRowUtil.MEASURE];
     return measures[index];
   }
 
   public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
 
-    return (byte[]) row[IgnoreDictionary.BYTE_ARRAY_INDEX_IN_ROW.getIndex()];
-  }
-
-  public static void prepareOutObj(Object[] out, Integer[] dimArray, byte[] byteBufferArr,
-      Object[] measureArray) {
-
-    out[IgnoreDictionary.DIMENSION_INDEX_IN_ROW.getIndex()] = dimArray;
-    out[IgnoreDictionary.BYTE_ARRAY_INDEX_IN_ROW.getIndex()] = byteBufferArr;
-    out[IgnoreDictionary.MEASURES_INDEX_IN_ROW.getIndex()] = measureArray;
-
+    return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
   }
 
   public static void prepareOutObj(Object[] out, int[] dimArray, byte[][] byteBufferArr,
       Object[] measureArray) {
 
-    out[IgnoreDictionary.DIMENSION_INDEX_IN_ROW.getIndex()] = dimArray;
-    out[IgnoreDictionary.BYTE_ARRAY_INDEX_IN_ROW.getIndex()] = byteBufferArr;
-    out[IgnoreDictionary.MEASURES_INDEX_IN_ROW.getIndex()] = measureArray;
-
-  }
-
-  /**
-   * This will extract the high cardinality count from the string.
-   */
-  public static int extractNoDictionaryCount(String noDictionaryDim) {
-    return extractNoDictionaryDimsArr(noDictionaryDim).length;
-  }
-
-  /**
-   * This method will split one single byte array of high card dims into array
-   * of byte arrays.
-   *
-   * @param NoDictionaryArr
-   * @param NoDictionaryCount
-   * @return
-   */
-  public static byte[][] splitNoDictionaryKey(byte[] NoDictionaryArr, int NoDictionaryCount)
{
-    byte[][] split = new byte[NoDictionaryCount][];
-
-    ByteBuffer buff = ByteBuffer.wrap(NoDictionaryArr, 2, NoDictionaryCount * 2);
-
-    int remainingCol = NoDictionaryCount;
-    short secIndex = 0;
-    short firstIndex = 0;
-    for (int i = 0; i < NoDictionaryCount; i++) {
-
-      if (remainingCol == 1) {
-        firstIndex = buff.getShort();
-        int length = NoDictionaryArr.length - firstIndex;
-
-        // add 2 bytes (short) as length required to determine size of
-        // each column value.
-
-        split[i] = new byte[length + 2];
-        ByteBuffer splittedCol = ByteBuffer.wrap(split[i]);
-        splittedCol.putShort((short) length);
-
-        System.arraycopy(NoDictionaryArr, firstIndex, split[i], 2, length);
-
-      } else {
-
-        firstIndex = buff.getShort();
-        secIndex = buff.getShort();
-        int length = secIndex - firstIndex;
-
-        // add 2 bytes (short) as length required to determine size of
-        // each column value.
-
-        split[i] = new byte[length + 2];
-        ByteBuffer splittedCol = ByteBuffer.wrap(split[i]);
-        splittedCol.putShort((short) length);
-
-        System.arraycopy(NoDictionaryArr, firstIndex, split[i], 2, length);
-        buff.position(buff.position() - 2);
-
-      }
-      remainingCol--;
-    }
-
-    return split;
-  }
-
-  /**
-   * @param index
-   * @param val
-   */
-  public static void setDimension(int index, int val, Object[] objArr) {
-    Integer[] dimensions = (Integer[]) objArr[IgnoreDictionary.DIMENSION_INDEX_IN_ROW.getIndex()];
-
-    dimensions[index] = val;
-
-  }
-
-  /**
-   * This will extract the high cardinality count from the string.
-   */
-  public static String[] extractNoDictionaryDimsArr(String noDictionaryDim) {
-
-    if (null == noDictionaryDim || noDictionaryDim.isEmpty()) {
-      return new String[0];
-    }
-
-    String[] NoDictionary = noDictionaryDim.split(CarbonCommonConstants.COMA_SPC_CHARACTER);
-    List<String> list1 = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-    for (int i = 0; i < NoDictionary.length; i++) {
-      String[] dim = NoDictionary[i].split(CarbonCommonConstants.COLON_SPC_CHARACTER);
-      list1.add(dim[0]);
-    }
-
-    return list1.toArray(new String[list1.size()]);
-  }
-  /**
-   * This will extract the high cardinality count from the string.
-   */
-  public static Map<String, String> extractDimColsDataTypeValues(String colDataTypes)
{
-    Map<String, String> mapOfColNameDataType =
-        new HashMap<String, String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    if (null == colDataTypes || colDataTypes.isEmpty()) {
-      return mapOfColNameDataType;
-    }
-    String[] colArray = colDataTypes.split(CarbonCommonConstants.AMPERSAND_SPC_CHARACTER);
-    String[] colValueArray = null;
-    for (String colArrayVal : colArray) {
-      colValueArray = colArrayVal.split(CarbonCommonConstants.COMA_SPC_CHARACTER);
-      mapOfColNameDataType.put(colValueArray[0].toLowerCase(), colValueArray[1]);
-
-    }
-    return mapOfColNameDataType;
-  }
-
-  public static byte[] convertListByteArrToSingleArr(List<byte[]> noDictionaryValKeyList)
{
-    ByteBuffer[] buffArr = new ByteBuffer[noDictionaryValKeyList.size()];
-    int index = 0;
-    for (byte[] singleColVal : noDictionaryValKeyList) {
-      buffArr[index] = ByteBuffer.allocate(singleColVal.length);
-      buffArr[index].put(singleColVal);
-      buffArr[index++].rewind();
-    }
-
-    return NonDictionaryUtil.packByteBufferIntoSingleByteArray(buffArr);
-
-  }
-
-  /**
-   * This method will convert boolean [] to String with comma separated.
-   * This needs to be done as sort step meta only supports string types.
-   *
-   * @param noDictionaryDimsMapping boolean arr to represent which dims is no dictionary
-   * @return
-   */
-  public static String convertBooleanArrToString(Boolean[] noDictionaryDimsMapping) {
-    StringBuilder  builder = new StringBuilder();
-    int index = 0;
-    for (; index < noDictionaryDimsMapping.length ; index++) {
-      builder.append(noDictionaryDimsMapping[index]);
-      builder.append(CarbonCommonConstants.COMA_SPC_CHARACTER);
-    }
-    int lastIndex = builder.lastIndexOf(CarbonCommonConstants.COMA_SPC_CHARACTER);
-    return -1 != lastIndex ? builder.substring(0, lastIndex) : builder.toString();
-  }
-
-  /**
-   * This will convert string to boolean[].
-   *
-   * @param noDictionaryColMapping String representation of the boolean [].
-   * @return
-   */
-  public static boolean[] convertStringToBooleanArr(String noDictionaryColMapping) {
-
-    String[] splittedValue = null != noDictionaryColMapping ?
-        noDictionaryColMapping.split(CarbonCommonConstants.COMA_SPC_CHARACTER) :
-        new String[0];
-
-    // convert string[] to boolean []
-
-    boolean[] noDictionaryMapping = new boolean[splittedValue.length];
-    int index = 0;
-    for (String str : splittedValue) {
-      noDictionaryMapping[index++] = Boolean.parseBoolean(str);
-    }
+    out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
+    out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
+    out[WriteStepRowUtil.MEASURE] = measureArray;
 
-    return noDictionaryMapping;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/353272ef/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index 12dcd15..91cc195 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -465,6 +465,8 @@ public class StoreCreator {
 
       String metadataInstance = gsonObjectToWrite.toJson(listOfLoadFolderDetails.toArray());
       brWriter.write(metadataInstance);
+    } catch (Exception ex) {
+      throw ex;
     } finally {
       try {
         if (null != brWriter) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/353272ef/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
b/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
index 038ac03..ad923a4 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
@@ -160,32 +160,6 @@ public class ColGroupMinMaxTest {
 		}
 	}
 
-	@Ignore
-	@Test
-	public void testRowStoreMinMax() throws KeyGenException {
-
-		DataHolder[] dataHolders = getDataHolders(segmentProperties.getColumnGroupModel().getNoOfColumnStore(),
-				mdkeyData.length);
-		for (int i = 0; i < mdkeyData.length; i++) {
-			byte[][] split = segmentProperties.getFixedLengthKeySplitter().splitKey(mdkeyData[i]);
-			for (int j = 0; j < split.length; j++) {
-				dataHolders[j].addData(split[j], i);
-			}
-
-		}
-		ColGroupBlockStorage[] rowBlockStorage = new ColGroupBlockStorage[dataHolders.length];
-		for (int i = 0; i < dataHolders.length; i++) {
-
-			rowBlockStorage[i] = new ColGroupBlockStorage(dataHolders[i]);
-		}
-		int[][] columnGroup = segmentProperties.getColumnGroups();
-		for (int i = 0; i < dataHolders.length; i++) {
-			assertMinMax(colGrpMinMax[i].getMin(), rowBlockStorage[i].getMax(),
-					columnGroup[i]);
-		}
-
-	}
-
 	private void assertMinMax(byte[] min, byte[] max, int[] columnGroup)
 			throws KeyGenException {
 


Mime
View raw message