carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kumarvishal09 <...@git.apache.org>
Subject [GitHub] carbondata pull request #1185: [WIP]Fixed Concurrent table data loading unsa...
Date Wed, 19 Jul 2017 12:17:02 GMT
Github user kumarvishal09 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1185#discussion_r128228749
  
    --- Diff: core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
---
    @@ -75,76 +79,67 @@
     
       private MemoryAllocator allocator;
     
    -  private long minimumMemory;
    -
    -  // for debug purpose
    -  private Set<MemoryBlock> set = new HashSet<>();
    -
       private UnsafeMemoryManager(long totalMemory, MemoryAllocator allocator) {
         this.totalMemory = totalMemory;
         this.allocator = allocator;
    -    long numberOfCores = CarbonProperties.getInstance().getNumberOfCores();
    -    long sortMemoryChunkSize = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
    -    sortMemoryChunkSize = sortMemoryChunkSize * 1024 * 1024;
    -    long totalWorkingMemoryForAllThreads = sortMemoryChunkSize * numberOfCores;
    -    if (totalWorkingMemoryForAllThreads >= totalMemory) {
    -      throw new RuntimeException("Working memory should be less than total memory configured,
"
    -          + "so either reduce the loading threads or increase the memory size. "
    -          + "(Number of threads * number of threads) should be less than total unsafe
memory");
    -    }
    -    minimumMemory = totalWorkingMemoryForAllThreads;
    -    LOGGER.info("Memory manager is created with size " + totalMemory + " with " + allocator
    -        + " and minimum reserve memory " + minimumMemory);
    +    LOGGER
    +        .info("Working Memory manager is created with size " + totalMemory + " with "
+ allocator);
       }
     
    -  private synchronized MemoryBlock allocateMemory(long memoryRequested) {
    +  private synchronized MemoryBlock allocateMemory(long taskId, long memoryRequested)
{
         if (memoryUsed + memoryRequested <= totalMemory) {
           MemoryBlock allocate = allocator.allocate(memoryRequested);
           memoryUsed += allocate.size();
    -      if (LOGGER.isDebugEnabled()) {
    -        set.add(allocate);
    -        LOGGER.error("Memory block (" + allocate + ") is created with size "  + allocate.size()
+
    -            ". Total memory used " + memoryUsed + "Bytes, left " + getAvailableMemory()
+ "Bytes");
    +      Set<MemoryBlock> listOfMemoryBlock = taskIdToMemoryBlockMap.get(taskId);
    --- End diff --
    
    @ravipesala Debug Log or info log??


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message