carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-3124] Updated log message in UnsafeMemoryManager
Date Wed, 28 Nov 2018 12:58:05 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master afe2b669b -> 1bbae2657


[CARBONDATA-3124] Updated log message in UnsafeMemoryManager

This closes #2948


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1bbae265
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1bbae265
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1bbae265

Branch: refs/heads/master
Commit: 1bbae26571b9dfc2a35ab626bc377924ea3c57a0
Parents: afe2b66
Author: shardul-cr7 <shardulsingh22@gmail.com>
Authored: Fri Nov 23 18:45:44 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Wed Nov 28 18:27:53 2018 +0530

----------------------------------------------------------------------
 .../carbondata/core/memory/UnsafeMemoryManager.java     | 12 ++++--------
 docs/faq.md                                             |  8 +++-----
 2 files changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1bbae265/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index 725f003..493ec0b 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -103,7 +103,7 @@ public class UnsafeMemoryManager {
   private UnsafeMemoryManager(long totalMemory, MemoryType memoryType) {
     this.totalMemory = totalMemory;
     this.memoryType = memoryType;
-    LOGGER.info("offheap Working Memory manager is created with size " + totalMemory + "
with "
+    LOGGER.info("Offheap Working Memory manager is created with size " + totalMemory + "
with "
         + memoryType);
   }
 
@@ -128,8 +128,8 @@ public class UnsafeMemoryManager {
       // not adding on heap memory block to map as JVM will take care of freeing the memory
       memoryBlock = MemoryAllocator.HEAP.allocate(memoryRequested);
       if (LOGGER.isDebugEnabled()) {
-        LOGGER.debug(
-            String.format("Creating onheap working Memory block (%s) with size:", memoryBlock));
+        LOGGER.debug(String
+            .format("Creating onheap working Memory block (%d) with size: ", memoryBlock.size()));
       }
     }
     return memoryBlock;
@@ -143,7 +143,7 @@ public class UnsafeMemoryManager {
       getMemoryAllocator(memoryBlock.getMemoryType()).free(memoryBlock);
       memoryUsed -= memoryBlock.size();
       memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
-      if (LOGGER.isDebugEnabled()) {
+      if (LOGGER.isDebugEnabled() && memoryBlock.getMemoryType() == MemoryType.OFFHEAP)
{
         LOGGER.debug(String.format("Freeing offheap working memory block (%s) with size:
%d, "
                 + "current available memory is: %d", memoryBlock.toString(), memoryBlock.size(),
             totalMemory - memoryUsed));
@@ -178,10 +178,6 @@ public class UnsafeMemoryManager {
         taskId, memoryUsed, StringUtils.join(taskIdToOffheapMemoryBlockMap.keySet(), ", ")));
   }
 
-  public synchronized boolean isMemoryAvailable() {
-    return memoryUsed > totalMemory;
-  }
-
   public long getUsableMemory() {
     return totalMemory;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1bbae265/docs/faq.md
----------------------------------------------------------------------
diff --git a/docs/faq.md b/docs/faq.md
index 3ac9a0a..dbcda4f 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -216,20 +216,18 @@ TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai"))
 ## How to check LRU cache memory footprint?
 To observe the LRU cache memory footprint in the logs, configure the below properties in
log4j.properties file.
 ```
-log4j.logger.org.apache.carbondata.core.memory.UnsafeMemoryManager = DEBUG
 log4j.logger.org.apache.carbondata.core.cache.CarbonLRUCache = DEBUG
 ```
-These properties will enable the DEBUG log for the CarbonLRUCache and UnsafeMemoryManager
which will print the information of memory consumed using which the LRU cache size can be
decided. **Note:** Enabling the DEBUG log will degrade the query performance.
+This property will enable the DEBUG log for the CarbonLRUCache and UnsafeMemoryManager which
will print the information of memory consumed using which the LRU cache size can be decided.
**Note:** Enabling the DEBUG log will degrade the query performance. Ensure carbon.max.driver.lru.cache.size
is configured to observe the current cache size.
 
 **Example:**
 ```
-18/09/26 15:05:28 DEBUG UnsafeMemoryManager: pool-44-thread-1 Memory block (org.apache.carbondata.core.memory.MemoryBlock@21312095)
is created with size 10. Total memory used 413Bytes, left 536870499Bytes
 18/09/26 15:05:29 DEBUG CarbonLRUCache: main Required size for entry /home/target/store/default/stored_as_carbondata_table/Fact/Part0/Segment_0/0_1537954529044.carbonindexmerge
:: 181 Current cache size :: 0
-18/09/26 15:05:30 DEBUG UnsafeMemoryManager: main Freeing memory of size: 105available memory:
 536870836
-18/09/26 15:05:30 DEBUG UnsafeMemoryManager: main Freeing memory of size: 76available memory:
 536870912
 18/09/26 15:05:30 INFO CarbonLRUCache: main Removed entry from InMemory lru cache :: /home/target/store/default/stored_as_carbondata_table/Fact/Part0/Segment_0/0_1537954529044.carbonindexmerge
 ```
+**Note:** If  `Removed entry from InMemory LRU cache` are frequently observed in logs, you
may have to increase the configured LRU size.
 
+To observe the LRU cache from heap dump, check the heap used by CarbonLRUCache class.
 ## Getting tablestatus.lock issues When loading data
 
   **Symptom**


Mime
View raw message