kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [32/51] [partial] kylin git commit: KYLIN-1416 keep only website in document branch
Date Thu, 11 Feb 2016 12:50:11 GMT
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
deleted file mode 100644
index 0af7dfc..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.File;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.LookupBytesTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
-    @Before
-    public void setup() throws Exception {
-        createTestMetadata();
-    }
-
-    @After
-    public void after() throws Exception {
-        cleanupTestMetadata();
-    }
-
-    @Test
-    public void testBasic() throws Exception {
-        TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES");
-        TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings");
-        LookupBytesTable lookup;
-
-        System.out.println("============================================================================");
-
-        File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv");
-        lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new FileTable("file://"
+ f.getAbsolutePath(), 10));
-        lookup.dump();
-
-        System.out.println("============================================================================");
-
-        f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv");
-        lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id"
}, new FileTable("file://" + f.getAbsolutePath(), 36));
-        lookup.dump();
-
-        System.out.println("============================================================================");
-
-        ByteArray k1 = new ByteArray(Bytes.toBytes("533"));
-        ByteArray k2 = new ByteArray(Bytes.toBytes("0"));
-        Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2
});
-        System.out.println(lookup.getRow(key));
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
deleted file mode 100644
index 4647ef8..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.math.BigDecimal;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- * 
- */
-public class NumberDictionaryTest {
-
-    NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec();
-    Random rand = new Random();
-
-    @Test
-    public void testNumberEncode() {
-        checkCodec("12345", "00000000000012345");
-        checkCodec("12345.123", "00000000000012345.123");
-        checkCodec("-12345", "-9999999999987654;");
-        checkCodec("-12345.123", "-9999999999987654.876;");
-        checkCodec("0", "00000000000000000");
-        checkCodec("0.0", "00000000000000000.0");
-    }
-
-    private void checkCodec(String number, String code) {
-        assertEquals(code, encodeNumber(number));
-        assertEquals(number, decodeNumber(code));
-    }
-
-    private String decodeNumber(String code) {
-        byte[] buf = Bytes.toBytes(code);
-        System.arraycopy(buf, 0, codec.buf, 0, buf.length);
-        codec.bufOffset = 0;
-        codec.bufLen = buf.length;
-        int len = codec.decodeNumber(buf, 0);
-        return Bytes.toString(buf, 0, len);
-    }
-
-    private String encodeNumber(String number) {
-        byte[] num1 = Bytes.toBytes(number);
-        codec.encodeNumber(num1, 0, num1.length);
-        return Bytes.toString(codec.buf, codec.bufOffset, codec.bufLen);
-    }
-
-    @Test
-    public void testDictionary() {
-        int n = 100;
-
-        Set<BigDecimal> set = Sets.newHashSet();
-        NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new
StringBytesConverter());
-        for (int i = 0; i < n; i++) {
-            String num = randNumber();
-            if (set.add(new BigDecimal(num))) {
-                builder.addValue(num);
-            }
-        }
-
-        List<BigDecimal> sorted = Lists.newArrayList();
-        sorted.addAll(set);
-        Collections.sort(sorted);
-
-        // test exact match
-        NumberDictionary<String> dict = builder.build(0);
-        for (int i = 0; i < sorted.size(); i++) {
-            String dictNum = dict.getValueFromId(i);
-            System.out.println(sorted.get(i) + "\t" + dictNum);
-        }
-
-        for (int i = 0; i < sorted.size(); i++) {
-            String dictNum = dict.getValueFromId(i);
-            assertEquals(sorted.get(i), new BigDecimal(dictNum));
-        }
-
-        // test rounding
-        for (int i = 0; i < n; i++) {
-            String randStr = randNumber();
-            BigDecimal rand = new BigDecimal(randStr);
-            int binarySearch = Collections.binarySearch(sorted, rand);
-            if (binarySearch >= 0)
-                continue;
-            int insertion = -(binarySearch + 1);
-            int expectedLowerId = insertion - 1;
-            int expectedHigherId = insertion;
-            // System.out.println("-- " + randStr + ", " + expectedLowerId +
-            // ", " + expectedHigherId);
-
-            if (expectedLowerId < 0) {
-                try {
-                    dict.getIdFromValue(randStr, -1);
-                    fail();
-                } catch (IllegalArgumentException ex) {
-                    // expect
-                }
-            } else {
-                assertEquals(expectedLowerId, dict.getIdFromValue(randStr, -1));
-            }
-
-            if (expectedHigherId >= sorted.size()) {
-                try {
-                    dict.getIdFromValue(randStr, 1);
-                    fail();
-                } catch (IllegalArgumentException ex) {
-                    // expect
-                }
-            } else {
-                assertEquals(expectedHigherId, dict.getIdFromValue(randStr, 1));
-            }
-        }
-    }
-
-    private String randNumber() {
-        int digits1 = rand.nextInt(10);
-        int digits2 = rand.nextInt(3);
-        int sign = rand.nextInt(2);
-        if (digits1 == 0 && digits2 == 0) {
-            return randNumber();
-        }
-        StringBuilder buf = new StringBuilder();
-        if (sign == 1)
-            buf.append("-");
-        for (int i = 0; i < digits1; i++)
-            buf.append("" + rand.nextInt(10));
-        if (digits2 > 0) {
-            buf.append(".");
-            for (int i = 0; i < digits2; i++)
-                buf.append("" + rand.nextInt(9) + 1); // BigDecimal thinks 4.5
-                                                      // != 4.50, my god!
-        }
-        return buf.toString();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
deleted file mode 100644
index c8dddde..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- * 
- */
-public class SnapshotManagerTest extends HBaseMetadataTestCase {
-
-    SnapshotManager snapshotMgr;
-
-    @Before
-    public void setup() throws Exception {
-        createTestMetadata();
-        snapshotMgr = SnapshotManager.getInstance(getTestConfig());
-    }
-
-    @After
-    public void after() throws Exception {
-        cleanupTestMetadata();
-    }
-
-    @Test
-    public void basicTest() throws Exception {
-        String tableName = "EDW.TEST_SITES";
-        HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()),
tableName);
-        TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
-        String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
-
-        snapshotMgr.wipeoutCache();
-
-        SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
-
-        // compare hive & snapshot
-        TableReader hiveReader = hiveTable.getReader();
-        TableReader snapshotReader = snapshot.getReader();
-
-        while (true) {
-            boolean hiveNext = hiveReader.next();
-            boolean snapshotNext = snapshotReader.next();
-            assertEquals(hiveNext, snapshotNext);
-
-            if (hiveNext == false)
-                break;
-
-            String[] hiveRow = hiveReader.getRow();
-            String[] snapshotRow = snapshotReader.getRow();
-            assertArrayEquals(hiveRow, snapshotRow);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
deleted file mode 100644
index 0d1999a..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.FileTableReader;
-import org.junit.Test;
-
-/**
- * @author yangli9
- * 
- */
-public class TableReaderTest {
-
-    @Test
-    public void testBasicReader() throws IOException {
-        File f = new File("src/test/resources/dict/DW_SITES");
-        FileTableReader reader = new FileTableReader("file://" + f.getAbsolutePath(), FileTable.DELIM_AUTO,
10);
-        while (reader.next()) {
-            assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT,
]", Arrays.toString(reader.getRow()));
-            break;
-        }
-        reader.close();
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
deleted file mode 100644
index cb15731..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.TreeSet;
-
-import org.junit.Test;
-
-public class TrieDictionaryTest {
-
-    public static void main(String[] args) throws Exception {
-        InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
-        // InputStream is =
-        // Util.getPackageResourceAsStream(TrieDictionaryTest.class,
-        // "eng_com.dic");
-        ArrayList<String> str = loadStrings(is);
-        benchmarkStringDictionary(str);
-    }
-
-    @Test
-    public void partOverflowTest() {
-        ArrayList<String> str = new ArrayList<String>();
-        // str.add("");
-        str.add("part");
-        str.add("par");
-        str.add("partition");
-        str.add("party");
-        str.add("parties");
-        str.add("paint");
-        String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
+ "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
-                + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
-        System.out.println("The length of the long string is " + longStr.length());
-        str.add(longStr);
-
-        str.add("zzzzzz" + longStr);// another long string
-
-        TrieDictionaryBuilder<String> b = newDictBuilder(str);
-        TrieDictionary<String> dict = b.build(0);
-
-        TreeSet<String> set = new TreeSet<String>();
-        for (String s : str) {
-            set.add(s);
-        }
-
-        // test serialize
-        dict = testSerialize(dict);
-
-        // test basic id<==>value
-        Iterator<String> it = set.iterator();
-        int id = 0;
-        int previousId = -1;
-        for (; it.hasNext(); id++) {
-            String value = it.next();
-
-            // in case of overflow parts, there exist interpolation nodes
-            // they exist to make sure that any node's part is shorter than 255
-            int actualId = dict.getIdFromValue(value);
-            assertTrue(actualId >= id);
-            assertTrue(actualId > previousId);
-            previousId = actualId;
-
-            assertEquals(value, dict.getValueFromId(actualId));
-        }
-    }
-
-    @Test
-    public void emptyValueTest() {
-        ArrayList<String> str = new ArrayList<String>();
-        str.add("");
-        str.add("part");
-        str.add("par");
-        str.add("partition");
-        str.add("party");
-        str.add("parties");
-        str.add("paint");
-        testStringDictionary(str, null);
-    }
-
-    @Test
-    public void simpleTrieTest() {
-        ArrayList<String> str = new ArrayList<String>();
-        str.add("part");
-        str.add("part"); // meant to be dup
-        str.add("par");
-        str.add("partition");
-        str.add("party");
-        str.add("parties");
-        str.add("paint");
-
-        ArrayList<String> notFound = new ArrayList<String>();
-        notFound.add("");
-        notFound.add("p");
-        notFound.add("pa");
-        notFound.add("pb");
-        notFound.add("parti");
-        notFound.add("partz");
-        notFound.add("partyz");
-
-        testStringDictionary(str, notFound);
-    }
-
-    @Test
-    public void englishWordsTest() throws Exception {
-        InputStream is = new FileInputStream("src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic");
-        ArrayList<String> str = loadStrings(is);
-        testStringDictionary(str, null);
-    }
-
-    @Test
-    public void categoryNamesTest() throws Exception {
-        InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
-        ArrayList<String> str = loadStrings(is);
-        testStringDictionary(str, null);
-    }
-
-    private static void benchmarkStringDictionary(ArrayList<String> str) throws UnsupportedEncodingException
{
-        TrieDictionaryBuilder<String> b = newDictBuilder(str);
-        b.stats().print();
-        TrieDictionary<String> dict = b.build(0);
-
-        TreeSet<String> set = new TreeSet<String>();
-        for (String s : str) {
-            set.add(s);
-        }
-
-        // prepare id==>value array and value==>id map
-        HashMap<String, Integer> map = new HashMap<String, Integer>();
-        String[] strArray = new String[set.size()];
-        byte[][] array = new byte[set.size()][];
-        Iterator<String> it = set.iterator();
-        for (int id = 0; it.hasNext(); id++) {
-            String value = it.next();
-            map.put(value, id);
-            strArray[id] = value;
-            array[id] = value.getBytes("UTF-8");
-        }
-
-        // System.out.println("Dict size in bytes:  " +
-        // MemoryUtil.deepMemoryUsageOf(dict));
-        // System.out.println("Map size in bytes:   " +
-        // MemoryUtil.deepMemoryUsageOf(map));
-        // System.out.println("Array size in bytes: " +
-        // MemoryUtil.deepMemoryUsageOf(strArray));
-
-        // warm-up, said that code only got JIT after run 1k-10k times,
-        // following jvm options may help
-        // -XX:CompileThreshold=1500
-        // -XX:+PrintCompilation
-        benchmark("Warm up", dict, set, map, strArray, array);
-        benchmark("Benchmark", dict, set, map, strArray, array);
-    }
-
-    private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String>
set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
-        int n = set.size();
-        int times = 10 * 1000 * 1000 / n; // run 10 million lookups
-        int keep = 0; // make sure JIT don't OPT OUT function calls under test
-        byte[] valueBytes = new byte[dict.getSizeOfValue()];
-        long start;
-
-        // benchmark value==>id, via HashMap
-        System.out.println(msg + " HashMap lookup value==>id");
-        start = System.currentTimeMillis();
-        for (int i = 0; i < times; i++) {
-            for (int j = 0; j < n; j++) {
-                keep |= map.get(strArray[j]);
-            }
-        }
-        long timeValueToIdByMap = System.currentTimeMillis() - start;
-        System.out.println(timeValueToIdByMap);
-
-        // benchmark value==>id, via Dict
-        System.out.println(msg + " Dictionary lookup value==>id");
-        start = System.currentTimeMillis();
-        for (int i = 0; i < times; i++) {
-            for (int j = 0; j < n; j++) {
-                keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
-            }
-        }
-        long timeValueToIdByDict = System.currentTimeMillis() - start;
-        System.out.println(timeValueToIdByDict);
-
-        // benchmark id==>value, via Array
-        System.out.println(msg + " Array lookup id==>value");
-        start = System.currentTimeMillis();
-        for (int i = 0; i < times; i++) {
-            for (int j = 0; j < n; j++) {
-                keep |= strArray[j].length();
-            }
-        }
-        long timeIdToValueByArray = System.currentTimeMillis() - start;
-        System.out.println(timeIdToValueByArray);
-
-        // benchmark id==>value, via Dict
-        System.out.println(msg + " Dictionary lookup id==>value");
-        start = System.currentTimeMillis();
-        for (int i = 0; i < times; i++) {
-            for (int j = 0; j < n; j++) {
-                keep |= dict.getValueBytesFromId(j, valueBytes, 0);
-            }
-        }
-        long timeIdToValueByDict = System.currentTimeMillis() - start;
-        System.out.println(timeIdToValueByDict);
-
-        return keep;
-    }
-
-    private static void testStringDictionary(ArrayList<String> str, ArrayList<String>
notFound) {
-        TrieDictionaryBuilder<String> b = newDictBuilder(str);
-        int baseId = new Random().nextInt(100);
-        TrieDictionary<String> dict = b.build(baseId);
-
-        TreeSet<String> set = new TreeSet<String>();
-        for (String s : str) {
-            set.add(s);
-        }
-
-        // test serialize
-        dict = testSerialize(dict);
-
-        // test basic id<==>value
-        Iterator<String> it = set.iterator();
-        int id = baseId;
-        for (; it.hasNext(); id++) {
-            String value = it.next();
-            // System.out.println("checking " + id + " <==> " + value);
-
-            assertEquals(id, dict.getIdFromValue(value));
-            assertEquals(value, dict.getValueFromId(id));
-        }
-        if (notFound != null) {
-            for (String s : notFound) {
-                try {
-                    dict.getIdFromValue(s);
-                    fail("For not found value '" + s + "', IllegalArgumentException is expected");
-                } catch (IllegalArgumentException e) {
-                    // good
-                }
-            }
-        }
-
-        // test null value
-        int nullId = dict.getIdFromValue(null);
-        assertNull(dict.getValueFromId(nullId));
-        int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
-        assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
-        assertEquals(nullId, nullId2);
-    }
-
-    private static TrieDictionary<String> testSerialize(TrieDictionary<String>
dict) {
-        try {
-            ByteArrayOutputStream bout = new ByteArrayOutputStream();
-            DataOutputStream dataout = new DataOutputStream(bout);
-            dict.write(dataout);
-            dataout.close();
-            ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
-            DataInputStream datain = new DataInputStream(bin);
-            TrieDictionary<String> r = new TrieDictionary<String>();
-            r.readFields(datain);
-            datain.close();
-            return r;
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String>
str) {
-        TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new
StringBytesConverter());
-        for (String s : str)
-            b.addValue(s);
-        return b;
-    }
-
-    private static ArrayList<String> loadStrings(InputStream is) throws Exception {
-        ArrayList<String> r = new ArrayList<String>();
-        BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
-        try {
-            String word;
-            while ((word = reader.readLine()) != null) {
-                word = word.trim();
-                if (word.isEmpty() == false)
-                    r.add(word);
-            }
-        } finally {
-            reader.close();
-            is.close();
-        }
-        return r;
-    }
-
-    @Test
-    public void testSuperLongStringValue() {
-        String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
-
-        TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new
StringBytesConverter());
-        String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
-        String v2 = longPrefix + "xyz";
-
-        b.addValue(v1);
-        b.addValue(v2);
-        TrieDictionary<String> dict = b.build(0);
-        dict.dump(System.out);
-    }
-
-    @Test
-    public void testRounding() {
-        // see NumberDictionaryTest.testRounding();
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
deleted file mode 100644
index 3d7267d..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by shaoshi on 2/3/15.
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
-    private KylinConfig config = null;
-
-    private LookupTable lookupTable;
-
-    @Before
-    public void setup() throws Exception {
-        createTestMetadata();
-        config = KylinConfig.getInstanceFromEnv();
-        lookupTable = initLookupTable();
-    }
-
-    @After
-    public void tearDown() {
-        cleanupTestMetadata();
-    }
-
-    @Test
-    public void testScan() throws Exception {
-        List<String> values = new ArrayList<String>();
-        values.add("2012-01-24");
-        values.add("2012-12-30");
-        List<String> results = lookupTable.scan("CAL_DT", values, "YEAR_BEG_DT");
-
-        Assert.assertTrue(results.size() > 0);
-        for (String i : results) {
-            System.out.println(i);
-
-            Assert.assertEquals("2012-01-01", i);
-        }
-    }
-
-    @Test
-    public void testMapRange() throws Exception {
-        List<String> values = new ArrayList<String>();
-        values.add("2012-01-24");
-        values.add("2012-12-30");
-        Pair<String, String> results = lookupTable.mapRange("CAL_DT", "2012-01-24",
"2012-12-30", "QTR_BEG_DT");
-
-        Assert.assertTrue(results != null);
-        System.out.println("The first qtr_beg_dt is " + results.getFirst());
-        System.out.println("The last qtr_beg_dt is " + results.getSecond());
-
-        Assert.assertEquals("2012-01-01", results.getFirst());
-        Assert.assertEquals("2012-10-01", results.getSecond());
-    }
-
-    @Test
-    public void testMapValues() throws Exception {
-        Set<String> values = new HashSet<String>();
-        values.add("2012-01-24");
-        values.add("2012-12-30");
-        Set<String> results = lookupTable.mapValues("CAL_DT", values, "YEAR_BEG_DT");
-
-        Assert.assertTrue(results.size() == 1);
-        for (String i : results) {
-            System.out.println(i);
-
-            Assert.assertEquals("2012-01-01", i);
-        }
-    }
-
-    public LookupTable initLookupTable() throws Exception {
-
-        MetadataManager metaMgr = MetadataManager.getInstance(config);
-
-        String tableName = "EDW.TEST_CAL_DT";
-        String[] pkCols = new String[] { "CAL_DT" };
-        String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot";
-        SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
-        TableDesc tableDesc = metaMgr.getTableDesc(tableName);
-        LookupTable lt = new LookupStringTable(tableDesc, pkCols, snapshot);
-
-        System.out.println(lt);
-
-        return lt;
-    }
-
-    private SnapshotManager getSnapshotManager() {
-        return SnapshotManager.getInstance(config);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/resources/dict/DW_SITES
----------------------------------------------------------------------
diff --git a/dictionary/src/test/resources/dict/DW_SITES b/dictionary/src/test/resources/dict/DW_SITES
deleted file mode 100644
index 6f6af46..0000000
Binary files a/dictionary/src/test/resources/dict/DW_SITES and /dev/null differ


Mime
View raw message