carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [49/52] [partial] incubator-carbondata git commit: Renamed packages to org.apache.carbondata and fixed errors
Date Mon, 15 Aug 2016 07:09:34 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/common/src/test/java/org/carbondata/common/logging/ft/LoggingServiceTest_FT.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/carbondata/common/logging/ft/LoggingServiceTest_FT.java b/common/src/test/java/org/carbondata/common/logging/ft/LoggingServiceTest_FT.java
deleted file mode 100644
index 66d1e3f..0000000
--- a/common/src/test/java/org/carbondata/common/logging/ft/LoggingServiceTest_FT.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.common.logging.ft;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-
-import junit.framework.TestCase;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.MDC;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class LoggingServiceTest_FT extends TestCase {
-
-  private static LogService logger =
-      LogServiceFactory.getLogService(LoggingServiceTest_FT.class.getName());
-
-  @Before public void setUp() throws Exception {
-    MDC.put("MODULE", "Function Test");
-    MDC.put("USER_NAME", "testuser");
-    MDC.put("CLIENT_IP", "127.0.0.1");
-    MDC.put("OPERATRION", "log");
-  }
-
-  @Test public void testIsAuditFileCreated() {
-    File f = new File("./unibiaudit.log");
-    Assert.assertFalse(f.exists());
-  }
-
-  @Test public void testAudit() {
-
-    String expectedAuditLine =
-        "[main] AUDIT [com.huawei.iweb.platform.logging.ft.LoggingServiceTest_FT] 127.0.0.1 "
-            + "testuser Function Test log- audit message created";
-    logger.audit("audit message created");
-
-    LogManager.shutdown();
-
-    try {
-      FileInputStream fstream = new FileInputStream("./unibiaudit.log");
-      BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
-      String actualAuditLine = null;
-      String strLine = null;
-      while ((strLine = br.readLine()) != null) {
-        actualAuditLine = strLine;
-      }
-
-      System.out.println(actualAuditLine);
-
-      if (actualAuditLine != null) {
-        int index = actualAuditLine.indexOf("[main]");
-        actualAuditLine = actualAuditLine.substring(index);
-        Assert.assertEquals(expectedAuditLine, actualAuditLine);
-      } else {
-        Assert.assertTrue(false);
-      }
-    } catch (FileNotFoundException e) {
-      e.printStackTrace();
-      Assert.assertTrue(!false);
-    } catch (IOException e) {
-      e.printStackTrace();
-      Assert.assertTrue(false);
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/common/src/test/java/org/carbondata/common/logging/impl/AuditExtendedRollingFileAppenderTest_UT.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/carbondata/common/logging/impl/AuditExtendedRollingFileAppenderTest_UT.java b/common/src/test/java/org/carbondata/common/logging/impl/AuditExtendedRollingFileAppenderTest_UT.java
deleted file mode 100644
index 8aa82d8..0000000
--- a/common/src/test/java/org/carbondata/common/logging/impl/AuditExtendedRollingFileAppenderTest_UT.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.common.logging.impl;
-
-import org.carbondata.common.logging.impl.AuditExtendedRollingFileAppender;
-import org.carbondata.common.logging.impl.AuditLevel;
-
-import junit.framework.Assert;
-import mockit.Deencapsulation;
-import org.apache.log4j.Logger;
-import org.apache.log4j.spi.LoggingEvent;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class AuditExtendedRollingFileAppenderTest_UT {
-
-  private AuditExtendedRollingFileAppender rAppender = null;
-
-  @Before public void setUp() throws Exception {
-    rAppender = new AuditExtendedRollingFileAppender();
-    Deencapsulation.setField(rAppender, "fileName", "audit.log");
-    Deencapsulation.setField(rAppender, "maxBackupIndex", 1);
-    Deencapsulation.setField(rAppender, "maxFileSize", 1000L);
-
-  }
-
-  @After public void tearDown() throws Exception {
-
-  }
-
-  @Test public void testRollOver() {
-    rAppender.rollOver();
-    rAppender.rollOver();
-    rAppender.rollOver();
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testCleanLogs() {
-    final String startName = "audit";
-    final String folderPath = "./";
-    int maxBackupIndex = 1;
-
-    Deencapsulation.invoke(rAppender, "cleanLogs", startName, folderPath, maxBackupIndex);
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testSubAppendLoggingEvent() {
-    Logger logger = Logger.getLogger(this.getClass());
-    LoggingEvent event = new LoggingEvent(null, logger, 0L, AuditLevel.AUDIT, null, null);
-
-    Deencapsulation.setField(rAppender, "qw", null);
-    try {
-      rAppender.subAppend(event);
-    } catch (Exception e) {
-      //
-    }
-    Assert.assertTrue(true);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/common/src/test/java/org/carbondata/common/logging/impl/AuditLevelTest_UT.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/carbondata/common/logging/impl/AuditLevelTest_UT.java b/common/src/test/java/org/carbondata/common/logging/impl/AuditLevelTest_UT.java
deleted file mode 100644
index 4e29d2f..0000000
--- a/common/src/test/java/org/carbondata/common/logging/impl/AuditLevelTest_UT.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.common.logging.impl;
-
-import org.carbondata.common.logging.impl.AuditLevel;
-
-import junit.framework.TestCase;
-import org.apache.log4j.Level;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class AuditLevelTest_UT extends TestCase {
-
-  @Before public void setUp() throws Exception {
-  }
-
-  @After public void tearDown() throws Exception {
-  }
-
-  @Test public void testAuditLevel() {
-    assertEquals(AuditLevel.AUDIT.toInt(), 55000);
-  }
-
-  @Test public void testToLevelIntLevel() {
-    assertSame(AuditLevel.AUDIT, AuditLevel.toLevel(55000, Level.DEBUG));
-  }
-
-  @Test public void testToLevelStringLevel() {
-    assertSame(AuditLevel.AUDIT, AuditLevel.toLevel("AUDIT", Level.DEBUG));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/common/src/test/java/org/carbondata/common/logging/impl/ExtendedRollingFileAppenderTest_UT.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/carbondata/common/logging/impl/ExtendedRollingFileAppenderTest_UT.java b/common/src/test/java/org/carbondata/common/logging/impl/ExtendedRollingFileAppenderTest_UT.java
deleted file mode 100644
index 14ccd20..0000000
--- a/common/src/test/java/org/carbondata/common/logging/impl/ExtendedRollingFileAppenderTest_UT.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.common.logging.impl;
-
-import org.carbondata.common.logging.impl.AuditLevel;
-import org.carbondata.common.logging.impl.ExtendedRollingFileAppender;
-
-import junit.framework.Assert;
-import mockit.Deencapsulation;
-import org.apache.log4j.Logger;
-import org.apache.log4j.spi.LoggingEvent;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ExtendedRollingFileAppenderTest_UT {
-
-  private ExtendedRollingFileAppender rAppender = null;
-
-  @Before public void setUp() throws Exception {
-    rAppender = new ExtendedRollingFileAppender();
-    Deencapsulation.setField(rAppender, "fileName", "dummy.log");
-    Deencapsulation.setField(rAppender, "maxBackupIndex", 1);
-    Deencapsulation.setField(rAppender, "maxFileSize", 1000L);
-  }
-
-  @After public void tearDown() throws Exception {
-  }
-
-  @Test public void testRollOver() {
-    rAppender.rollOver();
-    rAppender.rollOver();
-    rAppender.rollOver();
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testCleanLogs() {
-    final String startName = "dummy";
-    final String folderPath = "./";
-    int maxBackupIndex = 1;
-
-    Deencapsulation.invoke(rAppender, "cleanLogs", startName, folderPath, maxBackupIndex);
-  }
-
-  @Test public void testSubAppendLoggingEvent() {
-    Logger logger = Logger.getLogger(this.getClass());
-    LoggingEvent event = new LoggingEvent(null, logger, 0L, AuditLevel.DEBUG, null, null);
-
-    try {
-      rAppender.subAppend(event);
-    } catch (Exception e) {
-      //
-    }
-    Assert.assertTrue(true);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/common/src/test/java/org/carbondata/common/logging/impl/FileUtilTest_UT.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/carbondata/common/logging/impl/FileUtilTest_UT.java b/common/src/test/java/org/carbondata/common/logging/impl/FileUtilTest_UT.java
deleted file mode 100644
index 900cd0e..0000000
--- a/common/src/test/java/org/carbondata/common/logging/impl/FileUtilTest_UT.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.common.logging.impl;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-
-import junit.framework.TestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class FileUtilTest_UT extends TestCase {
-
-  /**
-   * @throws Exception
-   */
-  @Before public void setUp() throws Exception {
-    File f = new File("myfile.txt");
-    if (!f.exists()) {
-      f.createNewFile();
-    }
-  }
-
-  /**
-   * @throws Exception
-   */
-  @After public void tearDown() throws Exception {
-    File f = new File("myfile.txt");
-    if (f.exists()) {
-      f.delete();
-    }
-  }
-
-  @Test public void testClose() {
-    try {
-      FileInputStream in = new FileInputStream(new File("myfile.txt"));
-      FileUtil.close(in);
-      assertTrue(true);
-    } catch (FileNotFoundException e) {
-      assertTrue(false);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/common/src/test/java/org/carbondata/common/logging/impl/StandardLogServiceTest_UT.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/carbondata/common/logging/impl/StandardLogServiceTest_UT.java b/common/src/test/java/org/carbondata/common/logging/impl/StandardLogServiceTest_UT.java
deleted file mode 100644
index 4c9f0a3..0000000
--- a/common/src/test/java/org/carbondata/common/logging/impl/StandardLogServiceTest_UT.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.common.logging.impl;
-
-import org.carbondata.common.logging.impl.StandardLogService;
-
-import junit.framework.TestCase;
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.log4j.Category;
-import org.apache.log4j.Priority;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class StandardLogServiceTest_UT extends TestCase {
-
-  private StandardLogService logService = null;
-
-  /**
-   * @throws Exception
-   */
-  @Before public void setUp() throws Exception {
-
-    new MockUp<Category>() {
-      @SuppressWarnings("unused")
-      @Mock public boolean isDebugEnabled() {
-        return true;
-      }
-
-      @SuppressWarnings("unused")
-      @Mock public boolean isEnabledFor(Priority level) {
-        return true;
-      }
-
-      @SuppressWarnings("unused")
-      @Mock public boolean isInfoEnabled() {
-        return true;
-      }
-    };
-
-    logService = new StandardLogService(this.getClass().getName());
-  }
-
-  /**
-   * @throws Exception
-   * @Author k00742797
-   * @Description : tearDown
-   */
-  @After public void tearDown() throws Exception {
-  }
-
-  @Test public void testStandardLogService() {
-    if (logService != null && logService instanceof StandardLogService) {
-      Assert.assertTrue(true);
-    } else {
-      Assert.assertTrue(false);
-    }
-  }
-
-  @Test public void testIsDebugEnabled() {
-    Assert.assertEquals(true, logService.isDebugEnabled());
-  }
-
-  @Test public void testIsWarnEnabled() {
-    Assert.assertEquals(true, logService.isWarnEnabled());
-  }
-
-  @Test public void testSecureLogEventObjectArray() {
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testAuditLogEventObjectArray() {
-    logService.audit("testing");
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testDebugLogEventObjectArray() {
-    logService.debug("testing");
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testErrorLogEventObjectArray() {
-    logService.error("testing");
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testErrorLogEventThrowableObjectArray() {
-    Exception exception = new Exception("test");
-    logService.error(exception);
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testErrorLogEventThrowableMessage() {
-    Exception exception = new Exception("test");
-    logService.error(exception, "additional message");
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testInfoLogEventObjectArray() {
-    logService.info("testing");
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testIsInfoEnabled() {
-    Assert.assertEquals(true, logService.isInfoEnabled());
-  }
-
-  @Test public void testDeleteLogs() {
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testFlushLogs() {
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testSetEventProperties() {
-    logService.setEventProperties("CLIENT_IP", "127.0.0.1");
-    Assert.assertTrue(true);
-  }
-
-  @Test public void testIsDoLog() {
-    StandardLogService.setDoLog(true);
-    Assert.assertEquals(true, StandardLogService.isDoLog());
-
-    StandardLogService.setDoLog(false);
-    Assert.assertEquals(false, StandardLogService.isDoLog());
-
-  }
-
-  @Test public void testSetDoLog() {
-    StandardLogService.setDoLog(true);
-    Assert.assertEquals(true, StandardLogService.isDoLog());
-  }
-
-  @Test public void testAuditString() {
-    logService.audit("audit message");
-    Assert.assertTrue(true);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java b/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java
new file mode 100644
index 0000000..577a5ed
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.common.ext;
+
+import java.util.UUID;
+
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.service.ColumnUniqueIdService;
+
+/**
+ * It returns unique id given column
+ */
+public class ColumnUniqueIdGenerator implements ColumnUniqueIdService {
+
+  private static ColumnUniqueIdService columnUniqueIdService = new ColumnUniqueIdGenerator();
+
+  @Override public String generateUniqueId(String databaseName, ColumnSchema columnSchema) {
+    return UUID.randomUUID().toString();
+  }
+
+  public static ColumnUniqueIdService getInstance() {
+    return columnUniqueIdService;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java
new file mode 100644
index 0000000..3cf1ad5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.common.ext;
+
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
+import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
+import org.apache.carbondata.core.reader.CarbonDictionaryReader;
+import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
+import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
+import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
+import org.apache.carbondata.core.service.DictionaryService;
+import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
+import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
+import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
+import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
+
+/**
+ * service to get dictionary reader and writer
+ */
+public class DictionaryFactory implements DictionaryService {
+
+  private static DictionaryService dictService = new DictionaryFactory();
+
+  /**
+   * get dictionary writer
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   * @param carbonStorePath
+   * @return
+   */
+  @Override public CarbonDictionaryWriter getDictionaryWriter(
+      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+      String carbonStorePath) {
+    return new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
+  }
+
+  /**
+   * get dictionary sort index writer
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   * @param carbonStorePath
+   * @return
+   */
+  @Override public CarbonDictionarySortIndexWriter getDictionarySortIndexWriter(
+      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+      String carbonStorePath) {
+    return new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier,
+        carbonStorePath);
+  }
+
+  /**
+   * get dictionary metadata reader
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   * @param carbonStorePath
+   * @return
+   */
+  @Override public CarbonDictionaryMetadataReader getDictionaryMetadataReader(
+      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+      String carbonStorePath) {
+    return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier,
+        columnIdentifier);
+  }
+
+  /**
+   * get dictionary reader
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   * @param carbonStorePath
+   * @return
+   */
+  @Override public CarbonDictionaryReader getDictionaryReader(
+      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+      String carbonStorePath) {
+    return new CarbonDictionaryReaderImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
+  }
+
+  /**
+   * get dictionary sort index reader
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   * @param carbonStorePath
+   * @return
+   */
+  @Override public CarbonDictionarySortIndexReader getDictionarySortIndexReader(
+      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+      String carbonStorePath) {
+    return new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier,
+        carbonStorePath);
+  }
+
+  public static DictionaryService getInstance() {
+    return dictService;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java b/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java
new file mode 100644
index 0000000..e5ff83a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.common.ext;
+
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+import org.apache.carbondata.core.carbon.path.CarbonStorePath;
+import org.apache.carbondata.core.carbon.path.CarbonTablePath;
+import org.apache.carbondata.core.service.PathService;
+
+/**
+ * Create helper to get path details
+ */
+public class PathFactory implements PathService {
+
+  private static PathService pathService = new PathFactory();
+
+  /**
+   * @param columnIdentifier
+   * @param storeLocation
+   * @param tableIdentifier
+   * @return store path related to tables
+   */
+  @Override public CarbonTablePath getCarbonTablePath(ColumnIdentifier columnIdentifier,
+      String storeLocation, CarbonTableIdentifier tableIdentifier) {
+    return CarbonStorePath.getCarbonTablePath(storeLocation, tableIdentifier);
+  }
+
+  public static PathService getInstance() {
+    return pathService;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java b/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java
new file mode 100644
index 0000000..09b4465
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.common.factory;
+
+import org.apache.carbondata.common.ext.ColumnUniqueIdGenerator;
+import org.apache.carbondata.common.ext.DictionaryFactory;
+import org.apache.carbondata.common.ext.PathFactory;
+import org.apache.carbondata.core.service.ColumnUniqueIdService;
+import org.apache.carbondata.core.service.DictionaryService;
+import org.apache.carbondata.core.service.PathService;
+
+/**
+ * Interface to get services
+ */
+public class CarbonCommonFactory {
+
+  /**
+   * @return dictionary service
+   */
+  public static DictionaryService getDictionaryService() {
+    return DictionaryFactory.getInstance();
+  }
+
+  /**
+   * @return path service
+   */
+  public static PathService getPathService() {
+    return PathFactory.getInstance();
+  }
+
+  /**
+   * @return unique id generator
+   */
+  public static ColumnUniqueIdService getColumnUniqueIdGenerator() {
+    return ColumnUniqueIdGenerator.getInstance();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
new file mode 100644
index 0000000..b519deb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache;
+
+import java.util.List;
+
+import org.apache.carbondata.core.util.CarbonUtilException;
+
+/**
+ * A semi-persistent mapping from keys to values. Cache entries are manually added using
+ * #get(Key), #getAll(List<Keys>) , and are stored in the cache until
+ * either evicted or manually invalidated.
+ * Implementations of this interface are expected to be thread-safe, and can be safely accessed
+ * by multiple concurrent threads.
+ */
+public interface Cache<K, V> {
+
+  /**
+   * This method will get the value for the given key. If value does not exist
+   * for the given key, it will check and load the value.
+   *
+   * @param key
+   * @return
+   * @throws CarbonUtilException in case memory is not sufficient to load data into memory
+   */
+  V get(K key) throws CarbonUtilException;
+
+  /**
+   * This method will return a list of values for the given list of keys.
+   * For each key, this method will check and load the data if required.
+   *
+   * @param keys
+   * @return
+   * @throws CarbonUtilException in case memory is not sufficient to load data into memory
+   */
+  List<V> getAll(List<K> keys) throws CarbonUtilException;
+
+  /**
+   * This method will return the value for the given key. It will not check and load
+   * the data for the given key
+   *
+   * @param key
+   * @return
+   */
+  V getIfPresent(K key);
+
+  /**
+   * This method will remove the cache for a given key
+   *
+   * @param key
+   */
+  void invalidate(K key);
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
new file mode 100644
index 0000000..fa505bf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
+import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * Cache provider class which will create a cache based on given type
+ */
+public class CacheProvider {
+
+  /**
+   * cache provider instance
+   */
+  private static CacheProvider cacheProvider = new CacheProvider();
+
+  /**
+   * a map that will hold the entry for cache type to cache object mapping
+   */
+  private Map<CacheType, Cache> cacheTypeToCacheMap =
+      new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+  /**
+   * a map that will hold the mapping of cache type to LRU cache instance
+   */
+  private Map<CacheType, CarbonLRUCache> cacheTypeToLRUCacheMap =
+      new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+  /**
+   * object lock instance to be used in synchronization block
+   */
+  private final Object lock = new Object();
+
+  /**
+   * private constructor to follow singleton design pattern for this class
+   */
+  private CacheProvider() {
+
+  }
+
+  /**
+   * @return cache provider instance
+   */
+  public static CacheProvider getInstance() {
+    return cacheProvider;
+  }
+
+  /**
+   * This method will check if a cache already exists for given cache type and create in case
+   * it is not present in the map
+   *
+   * @param cacheType       type of cache
+   * @param carbonStorePath store path
+   * @param <K>
+   * @param <V>
+   * @return
+   */
+  public <K, V> Cache<K, V> createCache(CacheType cacheType, String carbonStorePath) {
+    //check if lru cache is null, if null create one
+    //check if cache is null for given cache type, if null create one
+    if (!dictionaryCacheAlreadyExists(cacheType)) {
+      synchronized (lock) {
+        if (!dictionaryCacheAlreadyExists(cacheType)) {
+          if (null == cacheTypeToLRUCacheMap.get(cacheType)) {
+            createLRULevelCacheInstance(cacheType);
+          }
+          createDictionaryCacheForGivenType(cacheType, carbonStorePath);
+        }
+      }
+    }
+    return cacheTypeToCacheMap.get(cacheType);
+  }
+
+  /**
+   * This method will create the cache for given cache type
+   *
+   * @param cacheType       type of cache
+   * @param carbonStorePath store path
+   */
+  private void createDictionaryCacheForGivenType(CacheType cacheType, String carbonStorePath) {
+    Cache cacheObject = null;
+    if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
+      cacheObject =
+          new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath,
+              cacheTypeToLRUCacheMap.get(cacheType));
+    } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
+      cacheObject =
+          new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath,
+              cacheTypeToLRUCacheMap.get(cacheType));
+    }
+    cacheTypeToCacheMap.put(cacheType, cacheObject);
+  }
+
+  /**
+   * This method will create the lru cache instance based on the given type
+   *
+   * @param cacheType
+   */
+  private void createLRULevelCacheInstance(CacheType cacheType) {
+    CarbonLRUCache carbonLRUCache = null;
+    // if cache type is dictionary cache, then same lru cache instance has to be shared
+    // between forward and reverse cache
+    if (cacheType.equals(CacheType.REVERSE_DICTIONARY) || cacheType
+        .equals(CacheType.FORWARD_DICTIONARY)) {
+      carbonLRUCache = new CarbonLRUCache(CarbonCommonConstants.CARBON_MAX_LEVEL_CACHE_SIZE,
+          CarbonCommonConstants.CARBON_MAX_LEVEL_CACHE_SIZE_DEFAULT);
+      cacheTypeToLRUCacheMap.put(CacheType.REVERSE_DICTIONARY, carbonLRUCache);
+      cacheTypeToLRUCacheMap.put(CacheType.FORWARD_DICTIONARY, carbonLRUCache);
+    }
+  }
+
+  /**
+   * This method will check whether the map already has an entry for
+   * given cache type
+   *
+   * @param cacheType
+   * @return
+   */
+  private boolean dictionaryCacheAlreadyExists(CacheType cacheType) {
+    return null != cacheTypeToCacheMap.get(cacheType);
+  }
+
+  /**
+   * Below method will be used to clear the cache
+   */
+  public void dropAllCache() {
+    cacheTypeToLRUCacheMap.clear();
+    cacheTypeToCacheMap.clear();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
new file mode 100644
index 0000000..ea511e9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+
+/**
+ * class which defines different cache types. cache type can be dictionary cache for
+ * forward (surrogate key to byte array mapping) and reverse (byte array to
+ * surrogate mapping) dictionary or a B-tree cache
+ */
+public class CacheType<K, V> {
+
+  /**
+   * Forward dictionary cache which maintains surrogate key to byte array mapping
+   */
+  public static final CacheType<DictionaryColumnUniqueIdentifier, Dictionary> FORWARD_DICTIONARY =
+      new CacheType("forward_dictionary");
+
+  /**
+   * Reverse dictionary cache which maintains byte array to surrogate key mapping
+   */
+  public static final CacheType<DictionaryColumnUniqueIdentifier, Dictionary> REVERSE_DICTIONARY =
+      new CacheType("reverse_dictionary");
+
+  /**
+   * cacheName which is unique name for a cache
+   */
+  private String cacheName;
+
+  /**
+   * @param cacheName
+   */
+  private CacheType(String cacheName) {
+    this.cacheName = cacheName;
+  }
+
+  /**
+   * @return cache unique name
+   */
+  public String getCacheName() {
+    return cacheName;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/Cacheable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/Cacheable.java b/core/src/main/java/org/apache/carbondata/core/cache/Cacheable.java
new file mode 100644
index 0000000..1259fe3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/Cacheable.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache;
+
+/**
+ * interface which declares methods which will decide whether to keep
+ * cacheable objects in memory
+ */
+public interface Cacheable {
+
+  /**
+   * This method will return the timestamp of file based on which decision
+   * the decision will be taken whether to read that file or not
+   *
+   * @return
+   */
+  long getFileTimeStamp();
+
+  /**
+   * This method will return the access count for a column based on which decision will be taken
+   * whether to keep the object in memory
+   *
+   * @return
+   */
+  int getAccessCount();
+
+  /**
+   * This method will return the memory size of a column
+   *
+   * @return
+   */
+  long getMemorySize();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
new file mode 100644
index 0000000..4ba38e4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * class which manages the lru cache
+ */
+public final class CarbonLRUCache {
+  /**
+   * constant for converting MB into bytes
+   */
+  private static final int BYTE_CONVERSION_CONSTANT = 1024 * 1024;
+  /**
+   * Attribute for Carbon LOGGER
+   */
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(CarbonLRUCache.class.getName());
+  /**
+   * Map that will contain key as table unique name and value as cache Holder
+   * object
+   */
+  private Map<String, Cacheable> lruCacheMap;
+  /**
+   * lruCacheSize
+   */
+  private long lruCacheMemorySize;
+  /**
+   * totalSize size of the cache
+   */
+  private long currentSize;
+
+  /**
+   * @param propertyName        property name to take the size configured
+   * @param defaultPropertyName default property in case size is not configured
+   */
+  public CarbonLRUCache(String propertyName, String defaultPropertyName) {
+    try {
+      lruCacheMemorySize = Integer
+          .parseInt(CarbonProperties.getInstance().getProperty(propertyName, defaultPropertyName));
+    } catch (NumberFormatException e) {
+      lruCacheMemorySize = Integer.parseInt(defaultPropertyName);
+    }
+    initCache();
+    if (lruCacheMemorySize > 0) {
+      LOGGER.info("Configured level cahce size is " + lruCacheMemorySize + " MB");
+      // convert in bytes
+      lruCacheMemorySize = lruCacheMemorySize * BYTE_CONVERSION_CONSTANT;
+    } else {
+      LOGGER.info("Column cache size not configured. Therefore default behavior will be "
+              + "considered and no LRU based eviction of columns will be done");
+    }
+  }
+
+  /**
+   * initialize lru cache
+   */
+  private void initCache() {
+    lruCacheMap =
+        new LinkedHashMap<String, Cacheable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE, 1.0f,
+            true);
+  }
+
+  /**
+   * This method will give the list of all the keys that can be deleted from
+   * the level LRU cache
+   */
+  private List<String> getKeysToBeRemoved(long size) {
+    List<String> toBeDeletedKeys =
+        new ArrayList<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+    long removedSize = 0;
+    for (Entry<String, Cacheable> entry : lruCacheMap.entrySet()) {
+      String key = entry.getKey();
+      Cacheable cacheInfo = entry.getValue();
+      long memorySize = cacheInfo.getMemorySize();
+      if (canBeRemoved(cacheInfo)) {
+        removedSize = removedSize + memorySize;
+        toBeDeletedKeys.add(key);
+        // check if after removing the current file size, required
+        // size when added to current size is sufficient to load a
+        // level or not
+        if (lruCacheMemorySize >= (currentSize - memorySize + size)) {
+          toBeDeletedKeys.clear();
+          toBeDeletedKeys.add(key);
+          removedSize = memorySize;
+          break;
+        }
+        // check if after removing the added size/removed size,
+        // required size when added to current size is sufficient to
+        // load a level or not
+        else if (lruCacheMemorySize >= (currentSize - removedSize + size)) {
+          break;
+        }
+      }
+    }
+    // this case will come when iteration is complete over the keys but
+    // still size is not sufficient for level file to be loaded, then we
+    // will not delete any of the keys
+    if ((currentSize - removedSize + size) > lruCacheMemorySize) {
+      toBeDeletedKeys.clear();
+    }
+    return toBeDeletedKeys;
+  }
+
+  /**
+   * @param cacheInfo
+   * @return
+   */
+  private boolean canBeRemoved(Cacheable cacheInfo) {
+    if (cacheInfo.getAccessCount() > 0) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * @param key
+   */
+  public void remove(String key) {
+    synchronized (lruCacheMap) {
+      removeKey(key);
+    }
+  }
+
+  /**
+   * This method will remove the key from lru cache
+   *
+   * @param key
+   */
+  private void removeKey(String key) {
+    Cacheable cacheable = lruCacheMap.get(key);
+    if (null != cacheable) {
+      currentSize = currentSize - cacheable.getMemorySize();
+    }
+    lruCacheMap.remove(key);
+    LOGGER.info("Removed level entry from InMemory level lru cache :: " + key);
+  }
+
+  /**
+   * This method will check if required size is available in the memory and then add
+   * the given cacheable to object to lru cache
+   *
+   * @param columnIdentifier
+   * @param cacheInfo
+   */
+  public boolean put(String columnIdentifier, Cacheable cacheInfo, long requiredSize) {
+    boolean columnKeyAddedSuccessfully = false;
+    if (freeMemorySizeForAddingCache(requiredSize)) {
+      synchronized (lruCacheMap) {
+        currentSize = currentSize + requiredSize;
+        if (null == lruCacheMap.get(columnIdentifier)) {
+          lruCacheMap.put(columnIdentifier, cacheInfo);
+        }
+        columnKeyAddedSuccessfully = true;
+      }
+      LOGGER.debug("Added level entry to InMemory level lru cache :: " + columnIdentifier);
+    } else {
+      LOGGER.error("Size not available. Column cannot be added to level lru cache :: "
+          + columnIdentifier + " .Required Size = " + requiredSize + " Size available "
+          + (lruCacheMemorySize - currentSize));
+    }
+    return columnKeyAddedSuccessfully;
+  }
+
+  /**
+   * This method will check a required column can be loaded into memory or not. If required
+   * this method will call for eviction of existing data from memory
+   *
+   * @param requiredSize
+   * @return
+   */
+  private boolean freeMemorySizeForAddingCache(long requiredSize) {
+    boolean memoryAvailable = false;
+    if (lruCacheMemorySize > 0) {
+      if (isSizeAvailableToLoadColumnDictionary(requiredSize)) {
+        memoryAvailable = true;
+      } else {
+        synchronized (lruCacheMap) {
+          // get the keys that can be removed from memory
+          List<String> keysToBeRemoved = getKeysToBeRemoved(requiredSize);
+          for (String cacheKey : keysToBeRemoved) {
+            removeKey(cacheKey);
+          }
+          // after removing the keys check again if required size is available
+          if (isSizeAvailableToLoadColumnDictionary(requiredSize)) {
+            memoryAvailable = true;
+          }
+        }
+      }
+    } else {
+      memoryAvailable = true;
+    }
+    return memoryAvailable;
+  }
+
+  /**
+   * This method will check if size is available to laod dictionary into memory
+   *
+   * @param requiredSize
+   * @return
+   */
+  private boolean isSizeAvailableToLoadColumnDictionary(long requiredSize) {
+    return lruCacheMemorySize >= (currentSize + requiredSize);
+  }
+
+  /**
+   * @param key
+   * @return
+   */
+  public Cacheable get(String key) {
+    synchronized (lruCacheMap) {
+      return lruCacheMap.get(key);
+    }
+  }
+
+  /**
+   * This method will empty the level cache
+   */
+  public void clear() {
+    synchronized (lruCacheMap) {
+      lruCacheMap.clear();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
new file mode 100644
index 0000000..a62695c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * class that implements cacheable interface and methods specific to column dictionary
+ */
+public abstract class AbstractColumnDictionaryInfo implements DictionaryInfo {
+
+  /**
+   * list that will hold all the dictionary chunks for one column
+   */
+  protected List<List<byte[]>> dictionaryChunks = new CopyOnWriteArrayList<>();
+
+  /**
+   * minimum value of surrogate key, dictionary value key will start from count 1
+   */
+  protected static final int MINIMUM_SURROGATE_KEY = 1;
+
+  /**
+   * atomic integer to maintain the access count for a column access
+   */
+  protected AtomicInteger accessCount = new AtomicInteger();
+
+  /**
+   * file timestamp
+   */
+  protected long fileTimeStamp;
+
+  /**
+   * offset till where file is read
+   */
+  protected long offsetTillFileIsRead;
+
+  /**
+   * length of dictionary metadata file
+   */
+  private long dictionaryMetaFileLength;
+
+  /**
+   * This method will return the timestamp of file based on which decision
+   * the decision will be taken whether to read that file or not
+   *
+   * @return
+   */
+  @Override public long getFileTimeStamp() {
+    return fileTimeStamp;
+  }
+
+  /**
+   * This method will return the access count for a column based on which decision will be taken
+   * whether to keep the object in memory
+   *
+   * @return
+   */
+  @Override public int getAccessCount() {
+    return accessCount.get();
+  }
+
+  /**
+   * This method will return the memory size of a column
+   *
+   * @return
+   */
+  @Override public long getMemorySize() {
+    return offsetTillFileIsRead;
+  }
+
+  /**
+   * This method will increment the access count for a column by 1
+   * whenever a column is getting used in query or incremental data load
+   */
+  @Override public void incrementAccessCount() {
+    accessCount.incrementAndGet();
+  }
+
+  /**
+   * This method will decrement the access count for a column by 1
+   * whenever a column usage is complete
+   */
+  private void decrementAccessCount() {
+    if (accessCount.get() > 0) {
+      accessCount.decrementAndGet();
+    }
+  }
+
+  /**
+   * This method will update the end offset of file everytime a file is read
+   *
+   * @param offsetTillFileIsRead
+   */
+  @Override public void setOffsetTillFileIsRead(long offsetTillFileIsRead) {
+    this.offsetTillFileIsRead = offsetTillFileIsRead;
+  }
+
+  /**
+   * This method will update the timestamp of a file if a file is modified
+   * like in case of incremental load
+   *
+   * @param fileTimeStamp
+   */
+  @Override public void setFileTimeStamp(long fileTimeStamp) {
+    this.fileTimeStamp = fileTimeStamp;
+  }
+
+  /**
+   * The method return the list of dictionary chunks of a column
+   * Applications Scenario.
+   * For preparing the column Sort info while writing the sort index file.
+   *
+   * @return
+   */
+  @Override public DictionaryChunksWrapper getDictionaryChunks() {
+    DictionaryChunksWrapper chunksWrapper = new DictionaryChunksWrapper(dictionaryChunks);
+    return chunksWrapper;
+  }
+
+  /**
+   * This method will release the objects and set default value for primitive types
+   */
+  @Override public void clear() {
+    decrementAccessCount();
+  }
+
+  /**
+   * This method will find and return the sort index for a given dictionary id.
+   * Applicable scenarios:
+   * 1. Used in case of order by queries when data sorting is required
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSortedIndex(int surrogateKey) {
+    return 0;
+  }
+
+  /**
+   * dictionary metadata file length which will be set whenever we reload dictionary
+   * data from disk
+   *
+   * @param dictionaryMetaFileLength length of dictionary metadata file
+   */
+  @Override public void setDictionaryMetaFileLength(long dictionaryMetaFileLength) {
+    this.dictionaryMetaFileLength = dictionaryMetaFileLength;
+  }
+
+  /**
+   * Dictionary meta file offset which will be read to check whether length of dictionary
+   * meta file has been modified
+   *
+   * @return
+   */
+  @Override public long getDictionaryMetaFileLength() {
+    return dictionaryMetaFileLength;
+  }
+
+  /**
+   * This method will find and return the dictionary value from sorted index.
+   * Applicable scenarios:
+   * 1. Query final result preparation in case of order by queries:
+   * While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param sortedIndex sort index of dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueFromSortedIndex(int sortedIndex) {
+    return null;
+  }
+
+  /**
+   * This method will set the sort order index of a dictionary column.
+   * Sort order index if the index of dictionary values after they are sorted.
+   *
+   * @param sortOrderIndex
+   */
+  @Override public void setSortOrderIndex(List<Integer> sortOrderIndex) {
+  }
+
+  /**
+   * This method will set the sort reverse index of a dictionary column.
+   * Sort reverse index is the index of dictionary values before they are sorted.
+   *
+   * @param sortReverseOrderIndex
+   */
+  @Override public void setSortReverseOrderIndex(List<Integer> sortReverseOrderIndex) {
+  }
+
+  /**
+   * This method will find and return the dictionary value for a given surrogate key.
+   * Applicable scenarios:
+   * 1. Query final result preparation : While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueForKey(int surrogateKey) {
+    String dictionaryValue = null;
+    if (surrogateKey < MINIMUM_SURROGATE_KEY) {
+      return dictionaryValue;
+    }
+    byte[] dictionaryValueInBytes = getDictionaryBytesFromSurrogate(surrogateKey);
+    if (null != dictionaryValueInBytes) {
+      dictionaryValue = new String(dictionaryValueInBytes,
+          Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+    }
+    return dictionaryValue;
+  }
+
+  /**
+   * This method will find and return the dictionary value as byte array for a
+   * given surrogate key
+   *
+   * @param surrogateKey
+   * @return
+   */
+  protected byte[] getDictionaryBytesFromSurrogate(int surrogateKey) {
+    byte[] dictionaryValueInBytes = null;
+    int totalSizeOfDictionaryChunksTraversed = 0;
+    for (List<byte[]> oneDictionaryChunk : dictionaryChunks) {
+      totalSizeOfDictionaryChunksTraversed =
+          totalSizeOfDictionaryChunksTraversed + oneDictionaryChunk.size();
+      // skip the dictionary chunk till surrogate key is lesser than size of
+      // dictionary chunks traversed
+      if (totalSizeOfDictionaryChunksTraversed < surrogateKey) {
+        continue;
+      }
+      // lets say surrogateKey = 26, total size traversed is 28, dictionary chunk size = 12
+      // then surrogate position in dictionary chunk list is = 26 - (28-12) - 1 = 9
+      // -1 because list index starts from 0
+      int surrogatePositionInDictionaryChunk =
+          surrogateKey - (totalSizeOfDictionaryChunksTraversed - oneDictionaryChunk.size()) - 1;
+      dictionaryValueInBytes = oneDictionaryChunk.get(surrogatePositionInDictionaryChunk);
+      break;
+    }
+    return dictionaryValueInBytes;
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(String value) {
+    byte[] keyData = value.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+    return getSurrogateKey(keyData);
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
new file mode 100644
index 0000000..01dd269
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.carbon.path.CarbonTablePath;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
+import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
+import org.apache.carbondata.core.service.DictionaryService;
+import org.apache.carbondata.core.service.PathService;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.CarbonUtilException;
+
+/**
+ * Abstract class which implements methods common to reverse and forward dictionary cache
+ */
+public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
+    V extends Dictionary>
+    implements Cache<DictionaryColumnUniqueIdentifier, Dictionary> {
+
+  /**
+   * thread pool size to be used for dictionary data reading
+   */
+  protected int thread_pool_size;
+
+  /**
+   * LRU cache variable
+   */
+  protected CarbonLRUCache carbonLRUCache;
+
+  /**
+   * c store path
+   */
+  protected String carbonStorePath;
+
+  /**
+   * @param carbonStorePath
+   * @param carbonLRUCache
+   */
+  public AbstractDictionaryCache(String carbonStorePath, CarbonLRUCache carbonLRUCache) {
+    this.carbonStorePath = carbonStorePath;
+    this.carbonLRUCache = carbonLRUCache;
+    initThreadPoolSize();
+  }
+
+  /**
+   * This method will initialize the thread pool size to be used for creating the
+   * max number of threads for a job
+   */
+  private void initThreadPoolSize() {
+    try {
+      thread_pool_size = Integer.parseInt(CarbonProperties.getInstance()
+          .getProperty(CarbonCommonConstants.NUM_CORES_LOADING,
+              CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
+    } catch (NumberFormatException e) {
+      thread_pool_size = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
+    }
+  }
+
+  /**
+   * This method will check if dictionary and its metadata file exists for a given column
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return
+   */
+  protected boolean isFileExistsForGivenColumn(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+    PathService pathService = CarbonCommonFactory.getPathService();
+    CarbonTablePath carbonTablePath = pathService
+        .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getColumnIdentifier(), carbonStorePath,
+            dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier());
+
+    String dictionaryFilePath =
+        carbonTablePath.getDictionaryFilePath(dictionaryColumnUniqueIdentifier
+            .getColumnIdentifier().getColumnId());
+    String dictionaryMetadataFilePath =
+        carbonTablePath.getDictionaryMetaFilePath(dictionaryColumnUniqueIdentifier
+            .getColumnIdentifier().getColumnId());
+    // check if both dictionary and its metadata file exists for a given column
+    return CarbonUtil.isFileExists(dictionaryFilePath) && CarbonUtil
+        .isFileExists(dictionaryMetadataFilePath);
+  }
+
+  /**
+   * This method will read dictionary metadata file and return the dictionary meta chunks
+   *
+   * @param dictionaryColumnUniqueIdentifier
+   * @return list of dictionary metadata chunks
+   * @throws IOException read and close method throws IO exception
+   */
+  protected CarbonDictionaryColumnMetaChunk readLastChunkFromDictionaryMetadataFile(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
+    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
+    CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService
+        .getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(),
+            dictionaryColumnUniqueIdentifier.getColumnIdentifier(), carbonStorePath);
+
+    CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;
+    // read metadata file
+    try {
+      carbonDictionaryColumnMetaChunk =
+          columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();
+    } finally {
+      // close the metadata reader
+      columnMetadataReaderImpl.close();
+    }
+    return carbonDictionaryColumnMetaChunk;
+  }
+
+  /**
+   * This method will validate dictionary metadata file for any modification
+   *
+   * @param carbonFile
+   * @param fileTimeStamp
+   * @param endOffset
+   * @return
+   */
+  private boolean isDictionaryMetaFileModified(CarbonFile carbonFile, long fileTimeStamp,
+      long endOffset) {
+    return carbonFile.isFileModified(fileTimeStamp, endOffset);
+  }
+
+  /**
+   * This method will return the carbon file objetc based on its type (local, HDFS)
+   *
+   * @param dictionaryColumnUniqueIdentifier
+   * @return
+   */
+  private CarbonFile getDictionaryMetaCarbonFile(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+    PathService pathService = CarbonCommonFactory.getPathService();
+    CarbonTablePath carbonTablePath = pathService
+        .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getColumnIdentifier(), carbonStorePath,
+            dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier());
+    String dictionaryFilePath =
+        carbonTablePath.getDictionaryMetaFilePath(dictionaryColumnUniqueIdentifier
+            .getColumnIdentifier().getColumnId());
+    FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);
+    CarbonFile carbonFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);
+    return carbonFile;
+  }
+
+  /**
+   * This method will get the value for the given key. If value does not exist
+   * for the given key, it will check and load the value.
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @param dictionaryInfo
+   * @param lruCacheKey
+   * @param loadSortIndex                    read and load sort index file in memory
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  protected void checkAndLoadDictionaryData(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
+      DictionaryInfo dictionaryInfo, String lruCacheKey, boolean loadSortIndex)
+      throws CarbonUtilException {
+    try {
+      // read last segment dictionary meta chunk entry to get the end offset of file
+      CarbonFile carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
+      boolean dictionaryMetaFileModified =
+          isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
+              dictionaryInfo.getDictionaryMetaFileLength());
+      // if dictionary metadata file is modified then only read the last entry from dictionary
+      // meta file
+      if (dictionaryMetaFileModified) {
+        synchronized (dictionaryInfo) {
+          carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
+          dictionaryMetaFileModified =
+              isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
+                  dictionaryInfo.getDictionaryMetaFileLength());
+          // Double Check :
+          // if dictionary metadata file is modified then only read the last entry from dictionary
+          // meta file
+          if (dictionaryMetaFileModified) {
+            CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk =
+                readLastChunkFromDictionaryMetadataFile(dictionaryColumnUniqueIdentifier);
+            // required size will be size total size of file - offset till file is
+            // already read
+            long requiredSize =
+                carbonDictionaryColumnMetaChunk.getEnd_offset() - dictionaryInfo.getMemorySize();
+            if (requiredSize > 0) {
+              boolean columnAddedToLRUCache =
+                  carbonLRUCache.put(lruCacheKey, dictionaryInfo, requiredSize);
+              // if column is successfully added to lru cache then only load the
+              // dictionary data
+              if (columnAddedToLRUCache) {
+                // load dictionary data
+                loadDictionaryData(dictionaryInfo, dictionaryColumnUniqueIdentifier,
+                    dictionaryInfo.getMemorySize(), carbonDictionaryColumnMetaChunk.getEnd_offset(),
+                    loadSortIndex);
+                // set the end offset till where file is read
+                dictionaryInfo
+                    .setOffsetTillFileIsRead(carbonDictionaryColumnMetaChunk.getEnd_offset());
+                dictionaryInfo.setFileTimeStamp(carbonFile.getLastModifiedTime());
+                dictionaryInfo.setDictionaryMetaFileLength(carbonFile.getSize());
+              } else {
+                throw new CarbonUtilException(
+                    "Cannot load dictionary into memory. Not enough memory available");
+              }
+            }
+          }
+        }
+      }
+      // increment the column access count
+      incrementDictionaryAccessCount(dictionaryInfo);
+    } catch (IOException e) {
+      throw new CarbonUtilException(e.getMessage());
+    }
+  }
+
+  /**
+   * This method will prepare the lru cache key and return the same
+   *
+   * @param columnIdentifier
+   * @return
+   */
+  protected String getLruCacheKey(String columnIdentifier, CacheType cacheType) {
+    String lruCacheKey =
+        columnIdentifier + CarbonCommonConstants.UNDERSCORE + cacheType.getCacheName();
+    return lruCacheKey;
+  }
+
+  /**
+   * This method will check and load the dictionary file in memory for a given column
+   *
+   * @param dictionaryInfo                   holds dictionary information and data
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @param dictionaryChunkStartOffset       start offset from where dictionary file has to
+   *                                         be read
+   * @param dictionaryChunkEndOffset         end offset till where dictionary file has to
+   *                                         be read
+   * @param loadSortIndex
+   * @throws IOException
+   */
+  private void loadDictionaryData(DictionaryInfo dictionaryInfo,
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
+      long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+      throws IOException {
+    DictionaryCacheLoader dictionaryCacheLoader =
+        new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(),
+            carbonStorePath);
+    dictionaryCacheLoader
+        .load(dictionaryInfo, dictionaryColumnUniqueIdentifier.getColumnIdentifier(),
+            dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);
+  }
+
+  /**
+   * This method will increment the access count for a given dictionary column
+   *
+   * @param dictionaryInfo
+   */
+  protected void incrementDictionaryAccessCount(DictionaryInfo dictionaryInfo) {
+    dictionaryInfo.incrementAccessCount();
+  }
+
+  /**
+   * This method will update the dictionary acceess count which is required for its removal
+   * from column LRU cache
+   *
+   * @param dictionaryList
+   */
+  protected void clearDictionary(List<Dictionary> dictionaryList) {
+    for (Dictionary dictionary : dictionaryList) {
+      dictionary.clear();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
new file mode 100644
index 0000000..08d9bef
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * class that implements methods specific for dictionary data look up
+ */
+public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo {
+
+  /**
+   * index after members are sorted
+   */
+  private AtomicReference<List<Integer>> sortOrderReference =
+      new AtomicReference<List<Integer>>(new ArrayList<Integer>());
+
+  /**
+   * inverted index to retrieve the member
+   */
+  private AtomicReference<List<Integer>> sortReverseOrderReference =
+      new AtomicReference<List<Integer>>(new ArrayList<Integer>());
+
+  private DataType dataType;
+
+  public ColumnDictionaryInfo(DataType dataType) {
+    this.dataType = dataType;
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value as byte array
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(byte[] value) {
+    return getSurrogateKeyFromDictionaryValue(value);
+  }
+
+  /**
+   * This method will find and return the sort index for a given dictionary id.
+   * Applicable scenarios:
+   * 1. Used in case of order by queries when data sorting is required
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSortedIndex(int surrogateKey) {
+    if (surrogateKey > sortReverseOrderReference.get().size()
+        || surrogateKey < MINIMUM_SURROGATE_KEY) {
+      return -1;
+    }
+    // decrement surrogate key as surrogate key basically means the index in array list
+    // because surrogate key starts from 1 and index of list from 0, so it needs to be
+    // decremented by 1
+    return sortReverseOrderReference.get().get(surrogateKey - 1);
+  }
+
+  /**
+   * This method will find and return the dictionary value from sorted index.
+   * Applicable scenarios:
+   * 1. Query final result preparation in case of order by queries:
+   * While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param sortedIndex sort index of dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueFromSortedIndex(int sortedIndex) {
+    if (sortedIndex > sortReverseOrderReference.get().size()
+        || sortedIndex < MINIMUM_SURROGATE_KEY) {
+      return null;
+    }
+    // decrement surrogate key as surrogate key basically means the index in array list
+    // because surrogate key starts from 1, sort index will start form 1 and index
+    // of list from 0, so it needs to be decremented by 1
+    int surrogateKey = sortOrderReference.get().get(sortedIndex - 1);
+    return getDictionaryValueForKey(surrogateKey);
+  }
+
+  /**
+   * This method will add a new dictionary chunk to existing list of dictionary chunks
+   *
+   * @param dictionaryChunk
+   */
+  @Override public void addDictionaryChunk(List<byte[]> dictionaryChunk) {
+    dictionaryChunks.add(dictionaryChunk);
+  }
+
+  /**
+   * This method will set the sort order index of a dictionary column.
+   * Sort order index if the index of dictionary values after they are sorted.
+   *
+   * @param sortOrderIndex
+   */
+  @Override public void setSortOrderIndex(List<Integer> sortOrderIndex) {
+    sortOrderReference.set(sortOrderIndex);
+  }
+
+  /**
+   * This method will set the sort reverse index of a dictionary column.
+   * Sort reverse index is the index of dictionary values before they are sorted.
+   *
+   * @param sortReverseOrderIndex
+   */
+  @Override public void setSortReverseOrderIndex(List<Integer> sortReverseOrderIndex) {
+    sortReverseOrderReference.set(sortReverseOrderIndex);
+  }
+
+  /**
+   * This method will apply binary search logic to find the surrogate key for the
+   * given value
+   *
+   * @param key to be searched
+   * @return
+   */
+  private int getSurrogateKeyFromDictionaryValue(byte[] key) {
+    String filterKey = new String(key, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+    int low = 0;
+    List<Integer> sortedSurrogates = sortOrderReference.get();
+    int high = sortedSurrogates.size() - 1;
+    while (low <= high) {
+      int mid = (low + high) >>> 1;
+      int surrogateKey = sortedSurrogates.get(mid);
+      byte[] dictionaryValue = getDictionaryBytesFromSurrogate(surrogateKey);
+      int cmp = -1;
+      if (this.getDataType() != DataType.STRING) {
+        cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey,
+            this.getDataType());
+
+      } else {
+        cmp = ByteUtil.UnsafeComparer.INSTANCE.compareTo(dictionaryValue, key);
+      }
+      if (cmp < 0) {
+        low = mid + 1;
+      } else if (cmp > 0) {
+        high = mid - 1;
+      } else {
+        return surrogateKey; // key found
+      }
+    }
+    return 0;
+  }
+
+  /**
+   * This method will apply binary search logic to find the surrogate key for the
+   * given value
+   *
+   * @param byteValuesOfFilterMembers to be searched
+   * @param surrogates
+   * @return
+   */
+  public void getIncrementalSurrogateKeyFromDictionary(List<byte[]> byteValuesOfFilterMembers,
+      List<Integer> surrogates) {
+    List<Integer> sortedSurrogates = sortOrderReference.get();
+    int low = 0;
+    for (byte[] byteValueOfFilterMember : byteValuesOfFilterMembers) {
+      String filterKey = new String(byteValueOfFilterMember,
+          Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+      if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterKey)) {
+        surrogates.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+        continue;
+      }
+      int high = sortedSurrogates.size() - 1;
+      while (low <= high) {
+        int mid = (low + high) >>> 1;
+        int surrogateKey = sortedSurrogates.get(mid);
+        byte[] dictionaryValue = getDictionaryBytesFromSurrogate(surrogateKey);
+        int cmp = -1;
+        //fortify fix
+        if (null == dictionaryValue) {
+          cmp = -1;
+        } else if (this.getDataType() != DataType.STRING) {
+          cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey,
+              this.getDataType());
+
+        } else {
+          cmp =
+              ByteUtil.UnsafeComparer.INSTANCE.compareTo(dictionaryValue, byteValueOfFilterMember);
+        }
+        if (cmp < 0) {
+          low = mid + 1;
+        } else if (cmp > 0) {
+          high = mid - 1;
+        } else {
+
+          surrogates.add(surrogateKey);
+          low = mid;
+          break;
+        }
+      }
+    }
+    //Default value has to be added
+    if (surrogates.isEmpty()) {
+      surrogates.add(0);
+    }
+  }
+
+  private int compareFilterKeyWithDictionaryKey(String dictionaryVal, String memberVal,
+      DataType dataType) {
+    try {
+      switch (dataType) {
+        case SHORT:
+          return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
+        case INT:
+          return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
+        case DOUBLE:
+          return Double
+              .compare((Double.parseDouble(dictionaryVal)), (Double.parseDouble(memberVal)));
+        case LONG:
+          return Long.compare((Long.parseLong(dictionaryVal)), (Long.parseLong(memberVal)));
+        case BOOLEAN:
+          return Boolean
+              .compare((Boolean.parseBoolean(dictionaryVal)), (Boolean.parseBoolean(memberVal)));
+        case TIMESTAMP:
+          SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+              .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+                  CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+          Date dateToStr;
+          Date dictionaryDate;
+          dateToStr = parser.parse(memberVal);
+          dictionaryDate = parser.parse(dictionaryVal);
+          return dictionaryDate.compareTo(dateToStr);
+        case DECIMAL:
+          java.math.BigDecimal javaDecValForDictVal = new java.math.BigDecimal(dictionaryVal);
+          java.math.BigDecimal javaDecValForMemberVal = new java.math.BigDecimal(memberVal);
+          return javaDecValForDictVal.compareTo(javaDecValForMemberVal);
+        default:
+          return -1;
+      }
+    } catch (Exception e) {
+      //In all data types excluding String data type the null member will be the highest
+      //while doing search in dictioary when the member comparison happens with filter member
+      //which is also null member, since the parsing fails in other data type except string
+      //explicit comparison is required, is both are null member then system has to return 0.
+      if (memberVal.equals(dictionaryVal)) {
+        return 0;
+      }
+      return 1;
+    }
+  }
+
+  /**
+   * getDataType().
+   *
+   * @return
+   */
+  public DataType getDataType() {
+    return dataType;
+  }
+
+}


Mime
View raw message