Mailing-List: contact commits-help@carbondata.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@carbondata.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: chenliang613@apache.org
To: commits@carbondata.incubator.apache.org
Date: Thu, 30 Jun 2016 17:41:56 -0000
Message-Id: <2146bfb7b57943408a95ae374bd37aea@git.apache.org>
In-Reply-To: <686ca27020db4633bf5b1c883df9389f@git.apache.org>
References: <686ca27020db4633bf5b1c883df9389f@git.apache.org>
Subject: [09/50] [abbrv] incubator-carbondata git commit: [BUG]Column group
 with no dictionary and timestamp (#713)
archived-at: Thu, 30 Jun 2016 17:41:57 -0000

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e96de9f5/processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java b/processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
index 18b2bfe..4c991f5 100644
--- a/processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
+++ b/processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
@@ -1,17 +1,20 @@
 package org.carbondata.processing.store.colgroup;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Random;
 import java.util.Set;
+import java.util.UUID;
 
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.core.keygenerator.KeyGenException;
-import org.carbondata.core.keygenerator.KeyGenerator;
-import org.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.vo.ColumnGroupModel;
 import org.carbondata.processing.store.colgroup.ColumnDataHolder;
 import org.carbondata.processing.store.colgroup.DataHolder;
 import org.carbondata.processing.store.colgroup.ColGroupBlockStorage;
@@ -24,206 +27,213 @@ import org.junit.Test;
  * RowStore store min max test
  */
 public class ColGroupMinMaxTest {
-    /**
-     * colgrpmodel
-     */
-    ColumnGroupModel colGrpModel;
-    /**
-     * column splitter
-     */
-    ColumnarSplitter columnSplitter;
-    /**
-     * column groups
-     */
-    int[][] columnGroups;
-
-    /**
-     * surrogate key
-     */
-    int[][] data;
-    /**
-     * mdkey data
-     */
-    byte[][] mdkeyData;
-
-    /**
-     * min value of surrogates
-     */
-    int[] min;
-    /**
-     * max value of surrogates
-     */
-    int[] max;
-    private ColGroupMinMax[] colGrpMinMax;
-
-    @Before
-    public void setupBeforeClass() throws KeyGenException {
-	int[] dimLens = new int[] { 100000, 1000, 10, 100, 100, 10000, 1000,
-		10, 1000, 1 };
-	columnGroups = new int[][] { { 0, 1, 2 }, { 3, 4 }, { 5, 6 },
-		{ 7, 8, 9 } };
-	colGrpModel = CarbonUtil.getColGroupModel(dimLens, columnGroups);
-	columnSplitter = new MultiDimKeyVarLengthVariableSplitGenerator(
-		CarbonUtil.getDimensionBitLength(
-			colGrpModel.getColumnGroupCardinality(),
-			colGrpModel.getColumnSplit()),
-		colGrpModel.getColumnSplit());
-	KeyGenerator keyGenerator = (KeyGenerator) columnSplitter;
-	initColGrpMinMax();
-	Random random = new Random();
-	data = new int[1000][];
-	min = new int[dimLens.length];
-	Arrays.fill(min, Integer.MAX_VALUE);
-	max = new int[dimLens.length];
-	Arrays.fill(max, Integer.MIN_VALUE);
-	for (int i = 0; i < 1000; i++) {
-
-	    data[i] = new int[dimLens.length];
-	    for (int j = 0; j < data[i].length; j++) {
-		data[i][j] = random.nextInt(dimLens[j]);
-	    }
-	    setMinData(data[i]);
-	    setMaxData(data[i]);
-	    System.out.println(Arrays.toString(data[i]));
-	}
-	mdkeyData = new byte[1000][];
-	for (int i = 0; i < 1000; i++) {
-	    mdkeyData[i] = keyGenerator.generateKey(data[i]);
-	    evaluateColGrpMinMax(mdkeyData[i]);
-	}
-    }
-
-    private void evaluateColGrpMinMax(byte[] mdkey) {
 
-	for (int colGrp = 0; colGrp < colGrpModel.getColumnGroup().length; colGrp++) {
-	    if (colGrpModel.getColumnGroup()[colGrp].length > 0) {
-		colGrpMinMax[colGrp].add(mdkey);
-	    }
+	/**
+	 * column groups
+	 */
+	int[][] columnGroups;
+
+	/**
+	 * surrogate key
+	 */
+	int[][] data;
+	/**
+	 * mdkey data
+	 */
+	byte[][] mdkeyData;
+
+	/**
+	 * min value of surrogates
+	 */
+	int[] min;
+	/**
+	 * max value of surrogates
+	 */
+	int[] max;
+	private ColGroupMinMax[] colGrpMinMax;
+	
+	private SegmentProperties segmentProperties;
+
+	@Before
+	public void setupBeforeClass() throws KeyGenException {
+		int[] dimLens = new int[] { 100000, 1000, 10, 100, 100, 10000, 1000, 10,
+				1000, 1 };
+		columnGroups = new int[][] { { 0, 1, 2 }, { 3, 4 }, { 5, 6 }, { 7, 8, 9 } };
+		segmentProperties = getSegmentProperties(dimLens, columnGroups);
+		initColGrpMinMax();
+		Random random = new Random();
+		data = new int[1000][];
+		min = new int[dimLens.length];
+		Arrays.fill(min, Integer.MAX_VALUE);
+		max = new int[dimLens.length];
+		Arrays.fill(max, Integer.MIN_VALUE);
+		for (int i = 0; i < 1000; i++) {
+
+			data[i] = new int[dimLens.length];
+			for (int j = 0; j < data[i].length; j++) {
+				data[i][j] = random.nextInt(dimLens[j]);
+			}
+			setMinData(data[i]);
+			setMaxData(data[i]);
+			System.out.println(Arrays.toString(data[i]));
+		}
+		mdkeyData = new byte[1000][];
+		for (int i = 0; i < 1000; i++) {
+			mdkeyData[i] = segmentProperties.getDimensionKeyGenerator().generateKey(data[i]);
+			evaluateColGrpMinMax(mdkeyData[i]);
+		}
 	}
-    }
-
-    private void initColGrpMinMax() {
-	int[][] colGrps = colGrpModel.getColumnGroup();
-	colGrpMinMax = new ColGroupMinMax[colGrps.length];
-	for (int colGrp = 0; colGrp < colGrps.length; colGrp++) {
-	    if (colGrps[colGrp].length > 0) {
-		colGrpMinMax[colGrp] = new ColGroupMinMax(colGrpModel,
-			columnSplitter, colGrp);
-	    }
-	}
-    }
 
-    private void setMaxData(int[] data) {
-	for (int i = 0; i < max.length; i++) {
-	    if (max[i] < data[i]) {
-		max[i] = data[i];
-	    }
+	private SegmentProperties getSegmentProperties(int[] dimLens, int[][] columnGroups) {
+		List<ColumnSchema> columnSchemas = new ArrayList<>();
+		for(int i=0;i<columnGroups.length;i++) {
+			  for(int j=0;j<columnGroups[i].length;j++) {
+			  	  columnSchemas.add(getDimensionColumn(i+j,i));
+			  }
+			
+		}
+		return new SegmentProperties(columnSchemas, dimLens);
 	}
+	private ColumnSchema getDimensionColumn(int var , int groupId) {
+  ColumnSchema dimColumn = new ColumnSchema();
+  dimColumn.setColumnar(false);
+  dimColumn.setColumnName("IMEI"+var);
+  dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+  dimColumn.setDataType(DataType.STRING);
+  dimColumn.setDimensionColumn(true);
+  List<Encoding> encodeList =
+      new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+  encodeList.add(Encoding.DICTIONARY);
+  dimColumn.setEncodingList(encodeList);
+  dimColumn.setColumnGroup(0);
+  dimColumn.setNumberOfChild(0);
+  return dimColumn;
+}
 
-    }
+	private void evaluateColGrpMinMax(byte[] mdkey) {
 
-    private void setMinData(int[] data) {
-	for (int i = 0; i < min.length; i++) {
-	    if (min[i] > data[i]) {
-		min[i] = data[i];
-	    }
+		for (int colGrp = 0; colGrp < segmentProperties.getColumnGroups().length; colGrp++) {
+			if (segmentProperties.getColumnGroups()[colGrp].length > 0) {
+				colGrpMinMax[colGrp].add(mdkey);
+			}
+		}
 	}
-    }
 
-    @Test
-    public void testRowStoreMinMax() throws KeyGenException {
+	private void initColGrpMinMax() {
+		int[][] colGrps = segmentProperties.getColumnGroups();
+		colGrpMinMax = new ColGroupMinMax[colGrps.length];
+		for (int colGrp = 0; colGrp < colGrps.length; colGrp++) {
+			if (colGrps[colGrp].length > 0) {
+				colGrpMinMax[colGrp] = new ColGroupMinMax(segmentProperties,
+						colGrp);
+			}
+		}
+	}
 
-	DataHolder[] dataHolders = getDataHolders(
-		colGrpModel.getNoOfColumnStore(), mdkeyData.length);
-	for (int i = 0; i < mdkeyData.length; i++) {
-	    byte[][] split = columnSplitter.splitKey(mdkeyData[i]);
-	    for (int j = 0; j < split.length; j++) {
-		dataHolders[j].addData(split[j], i);
-	    }
+	private void setMaxData(int[] data) {
+		for (int i = 0; i < max.length; i++) {
+			if (max[i] < data[i]) {
+				max[i] = data[i];
+			}
+		}
 
 	}
-	ColGroupBlockStorage[] rowBlockStorage = new ColGroupBlockStorage[dataHolders.length];
-	for (int i = 0; i < dataHolders.length; i++) {
 
-	    rowBlockStorage[i] = new ColGroupBlockStorage(dataHolders[i]);
-	}
-	int[][] columnGroup = colGrpModel.getColumnGroup();
-	for (int i = 0; i < dataHolders.length; i++) {
-	    assertMinMax(colGrpMinMax[i].getMin(), rowBlockStorage[i].getMax(),
-		    columnGroup[i]);
+	private void setMinData(int[] data) {
+		for (int i = 0; i < min.length; i++) {
+			if (min[i] > data[i]) {
+				min[i] = data[i];
+			}
+		}
 	}
 
-    }
-
-    private void assertMinMax(byte[] min, byte[] max, int[] columnGroup)
-	    throws KeyGenException {
-	KeyGenerator keyGenerator = (KeyGenerator) columnSplitter;
-
-	int columnStartIndex = 0;
-	for (int i = 0; i < columnGroup.length; i++) {
-	    int col = columnGroup[i];
-	    int[] maskByteRange = getMaskByteRange(col);
-	    int[] maskBytePosition = new int[keyGenerator.getKeySizeInBytes()];
-	    updateMaskedKeyRanges(maskBytePosition, maskByteRange);
-
-	    byte[] columnMin = new byte[maskByteRange.length];
-	    System.arraycopy(min, columnStartIndex, columnMin, 0,
-		    maskByteRange.length);
-	    byte[] columnMax = new byte[maskByteRange.length];
-	    System.arraycopy(max, columnStartIndex, columnMax, 0,
-		    maskByteRange.length);
-
-	    long[] minKeyArray = keyGenerator.getKeyArray(columnMin,
-		    maskBytePosition);
-	    long[] maxKeyArray = keyGenerator.getKeyArray(columnMax,
-		    maskBytePosition);
-	    System.out.println("calculated:(min,max) for column " + col + ":("
-		    + minKeyArray[col] + "," + maxKeyArray[col] + ")");
-	    System.out.println("correct:(min,max) for column " + col + ":("
-		    + this.min[col] + "," + this.max[col] + ")");
-	    columnStartIndex += maskByteRange.length;
-	    Assert.assertEquals(minKeyArray[col], this.min[col]);
-	    Assert.assertEquals(maxKeyArray[col], this.max[col]);
+	@Test
+	public void testRowStoreMinMax() throws KeyGenException {
+
+		DataHolder[] dataHolders = getDataHolders(segmentProperties.getColumnGroupModel().getNoOfColumnStore(),
+				mdkeyData.length);
+		for (int i = 0; i < mdkeyData.length; i++) {
+			byte[][] split = segmentProperties.getFixedLengthKeySplitter().splitKey(mdkeyData[i]);
+			for (int j = 0; j < split.length; j++) {
+				dataHolders[j].addData(split[j], i);
+			}
+
+		}
+		ColGroupBlockStorage[] rowBlockStorage = new ColGroupBlockStorage[dataHolders.length];
+		for (int i = 0; i < dataHolders.length; i++) {
+
+			rowBlockStorage[i] = new ColGroupBlockStorage(dataHolders[i]);
+		}
+		int[][] columnGroup = segmentProperties.getColumnGroups();
+		for (int i = 0; i < dataHolders.length; i++) {
+			assertMinMax(colGrpMinMax[i].getMin(), rowBlockStorage[i].getMax(),
+					columnGroup[i]);
+		}
 
 	}
 
-    }
-
-    private DataHolder[] getDataHolders(int noOfColumn, int noOfRow) {
-	DataHolder[] dataHolders = new DataHolder[noOfColumn];
-	for (int colGrp = 0; colGrp < noOfColumn; colGrp++) {
-	    if (colGrpModel.isColumnar(colGrp)) {
-		dataHolders[colGrp] = new ColumnDataHolder(noOfRow);
-	    } else {
-		dataHolders[colGrp] = new ColGroupDataHolder(this.colGrpModel,
-			this.columnSplitter.getBlockKeySize()[colGrp], noOfRow,
-			colGrpMinMax[colGrp]);
-	    }
+	private void assertMinMax(byte[] min, byte[] max, int[] columnGroup)
+			throws KeyGenException {
+
+		int columnStartIndex = 0;
+		for (int i = 0; i < columnGroup.length; i++) {
+			int col = columnGroup[i];
+			int[] maskByteRange = getMaskByteRange(col);
+			int[] maskBytePosition = new int[segmentProperties.getDimensionKeyGenerator().getKeySizeInBytes()];
+			updateMaskedKeyRanges(maskBytePosition, maskByteRange);
+
+			byte[] columnMin = new byte[maskByteRange.length];
+			System.arraycopy(min, columnStartIndex, columnMin, 0, maskByteRange.length);
+			byte[] columnMax = new byte[maskByteRange.length];
+			System.arraycopy(max, columnStartIndex, columnMax, 0, maskByteRange.length);
+
+			long[] minKeyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(columnMin, maskBytePosition);
+			long[] maxKeyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(columnMax, maskBytePosition);
+			System.out.println("calculated:(min,max) for column " + col + ":("
+					+ minKeyArray[col] + "," + maxKeyArray[col] + ")");
+			System.out.println("correct:(min,max) for column " + col + ":("
+					+ this.min[col] + "," + this.max[col] + ")");
+			columnStartIndex += maskByteRange.length;
+			Assert.assertEquals(minKeyArray[col], this.min[col]);
+			Assert.assertEquals(maxKeyArray[col], this.max[col]);
+
+		}
+
 	}
-	return dataHolders;
-    }
-
-    private int[] getMaskByteRange(int col) {
-	KeyGenerator keyGenerator = (KeyGenerator) columnSplitter;
-	Set<Integer> integers = new HashSet<>();
-	int[] range = keyGenerator.getKeyByteOffsets(col);
-	for (int j = range[0]; j <= range[1]; j++) {
-	    integers.add(j);
+
+	private DataHolder[] getDataHolders(int noOfColumn, int noOfRow) {
+		DataHolder[] dataHolders = new DataHolder[noOfColumn];
+		for (int colGrp = 0; colGrp < noOfColumn; colGrp++) {
+			if (segmentProperties.getColumnGroupModel().isColumnar(colGrp)) {
+				dataHolders[colGrp] = new ColumnDataHolder(noOfRow);
+			} else {
+				dataHolders[colGrp] = new ColGroupDataHolder(
+						segmentProperties.getFixedLengthKeySplitter().getBlockKeySize()[colGrp], noOfRow,
+						colGrpMinMax[colGrp]);
+			}
+		}
+		return dataHolders;
 	}
-	int[] byteIndexs = new int[integers.size()];
-	int j = 0;
-	for (Iterator<Integer> iterator = integers.iterator(); iterator
-		.hasNext();) {
-	    Integer integer = (Integer) iterator.next();
-	    byteIndexs[j++] = integer.intValue();
+
+	private int[] getMaskByteRange(int col) {
+		Set<Integer> integers = new HashSet<>();
+		int[] range = segmentProperties.getDimensionKeyGenerator().getKeyByteOffsets(col);
+		for (int j = range[0]; j <= range[1]; j++) {
+			integers.add(j);
+		}
+		int[] byteIndexs = new int[integers.size()];
+		int j = 0;
+		for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext();) {
+			Integer integer = (Integer) iterator.next();
+			byteIndexs[j++] = integer.intValue();
+		}
+		return byteIndexs;
 	}
-	return byteIndexs;
-    }
 
-    private void updateMaskedKeyRanges(int[] maskedKey, int[] maskedKeyRanges) {
-	Arrays.fill(maskedKey, -1);
-	for (int i = 0; i < maskedKeyRanges.length; i++) {
-	    maskedKey[maskedKeyRanges[i]] = i;
+	private void updateMaskedKeyRanges(int[] maskedKey, int[] maskedKeyRanges) {
+		Arrays.fill(maskedKey, -1);
+		for (int i = 0; i < maskedKeyRanges.length; i++) {
+			maskedKey[maskedKeyRanges[i]] = i;
+		}
 	}
-    }
 }