kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [14/30] incubator-kylin git commit: KYLIN-1099 Remove dedup in dictionary value enumerator
Date Wed, 18 Nov 2015 02:39:08 GMT
KYLIN-1099 Remove dedup in dictionary value enumerator

Signed-off-by: Li, Yang <yangli9@ebay.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a441237a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a441237a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a441237a

Branch: refs/heads/1.x-HBase1.1.3
Commit: a441237afbf45736eea74cf6638e34de60a82cea
Parents: 882ca61
Author: lidongsjtu <dongli@ebay.com>
Authored: Mon Nov 2 09:55:05 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Wed Nov 11 10:18:13 2015 +0800

----------------------------------------------------------------------
 .../dict/ListDictionaryValueEnumerator.java     | 50 ++++++++++++++++++++
 .../dict/MultipleDictionaryValueEnumerator.java | 10 +---
 .../kylin/dict/TableColumnValueEnumerator.java  | 13 ++---
 .../lookup/ListDictionaryValueEnumerator.java   | 50 --------------------
 .../job/hadoop/cube/MergeCuboidMapperTest.java  |  2 +-
 5 files changed, 55 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
new file mode 100644
index 0000000..9ae08a8
--- /dev/null
+++ b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.dict.IDictionaryValueEnumerator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Created by dongli on 10/28/15.
+ */
+public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator {
+    ListIterator<byte[]> listIterator;
+
+    public ListDictionaryValueEnumerator(List<byte[]> list) {
+        listIterator = list.listIterator();
+    }
+
+    @Override
+    public byte[] current() throws IOException {
+        return listIterator.next();
+    }
+
+    @Override
+    public boolean moveNext() throws IOException {
+        return listIterator.hasNext();
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index 4cf72d3..13f7394 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -19,12 +19,9 @@
 package org.apache.kylin.dict;
 
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
 import org.apache.kylin.common.util.Bytes;
 
 import java.io.IOException;
-import java.util.HashSet;
 import java.util.List;
 
 /**
@@ -32,7 +29,6 @@ import java.util.List;
  */
 @SuppressWarnings("rawtypes")
 public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumerator {
-    private HashSet<byte[]> dedup = Sets.newHashSet();
     private int curDictIndex = 0;
     private Dictionary curDict;
     private int curKey;
@@ -69,11 +65,7 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
                 }
             }
 
-            if (dedup.contains(curValue)) {
-                return moveNext();
-            } else {
-                return true;
-            }
+            return true;
         }
         curValue = null;
         return false;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
index 64a0e39..42f5791 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
@@ -18,13 +18,11 @@
 
 package org.apache.kylin.dict;
 
-import com.google.common.collect.Sets;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dict.lookup.ReadableTable;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Set;
 
 /**
  * Created by dongli on 10/28/15.
@@ -34,7 +32,6 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator
{
     private ReadableTable.TableReader reader;
     private int colIndex;
     private byte[] colValue;
-    private Set<String> dedup = Sets.newHashSet();
 
     public TableColumnValueEnumerator(ReadableTable.TableReader reader, int colIndex) {
         this.reader = reader;
@@ -55,13 +52,9 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator
{
                 }
                 colStrValue = split[colIndex];
             }
-            if (!dedup.contains(colStrValue)) {
-                dedup.add(colStrValue);
-                colValue = Bytes.toBytes(colStrValue);
-                return true;
-            } else {
-                return moveNext();
-            }
+
+            colValue = Bytes.toBytes(colStrValue);
+            return true;
 
         } else {
             colValue = null;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
b/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
deleted file mode 100644
index 581ee43..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import org.apache.kylin.dict.IDictionaryValueEnumerator;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 10/28/15.
- */
-public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator {
-    ListIterator<byte[]> listIterator;
-
-    public ListDictionaryValueEnumerator(List<byte[]> list) {
-        listIterator = list.listIterator();
-    }
-
-    @Override
-    public byte[] current() throws IOException {
-        return listIterator.next();
-    }
-
-    @Override
-    public boolean moveNext() throws IOException {
-        return listIterator.hasNext();
-    }
-
-    @Override
-    public void close() throws IOException {
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index 7a2ebad..9a1fdfb 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -38,7 +38,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.lookup.ListDictionaryValueEnumerator;
+import org.apache.kylin.dict.ListDictionaryValueEnumerator;
 import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.TblColRef;


Mime
View raw message