orc-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omalley <...@git.apache.org>
Subject [GitHub] orc pull request #299: ORC-203 - Update StringStatistics to trim long string...
Date Tue, 07 Aug 2018 22:16:36 GMT
Github user omalley commented on a diff in the pull request:

    https://github.com/apache/orc/pull/299#discussion_r208393268
  
    --- Diff: java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java ---
    @@ -543,35 +551,73 @@ public void reset() {
           super.reset();
           minimum = null;
           maximum = null;
    +      isLowerBoundSet = false;
    +      isUpperBoundSet = false;
           sum = 0;
         }
     
         @Override
         public void updateString(Text value) {
           if (minimum == null) {
    -        maximum = minimum = new Text(value);
    +        if(value.getLength() > MAX_BYTES_RECORDED) {
    +         minimum = truncateLowerBound(value);
    +         maximum = truncateUpperBound(value);
    +         isLowerBoundSet = true;
    +         isUpperBoundSet = true;
    +        } else {
    +          maximum = minimum = new Text(value);
    +        }
           } else if (minimum.compareTo(value) > 0) {
    -        minimum = new Text(value);
    +        if(value.getLength() > MAX_BYTES_RECORDED) {
    +          minimum = truncateLowerBound(value);
    +          isLowerBoundSet = true;
    +        }else {
    +          minimum = new Text(value);
    +        }
           } else if (maximum.compareTo(value) < 0) {
    -        maximum = new Text(value);
    +        if(value.getLength() > MAX_BYTES_RECORDED) {
    +          maximum = truncateUpperBound(value);
    +          isUpperBoundSet = true;
    +        } else {
    +          maximum = new Text(value);
    +        }
           }
           sum += value.getLength();
         }
     
    +
         @Override
         public void updateString(byte[] bytes, int offset, int length,
                                  int repetitions) {
    +      byte[] input = Arrays.copyOfRange(bytes, offset, offset+(length));
           if (minimum == null) {
    -        maximum = minimum = new Text();
    -        maximum.set(bytes, offset, length);
    +        if(length > MAX_BYTES_RECORDED) {
    +          minimum = truncateLowerBound(input);
    +          maximum = truncateUpperBound(input);
    +          isLowerBoundSet = true;
    +          isUpperBoundSet = true;
    +        } else {
    +          maximum = minimum = new Text();
    +          maximum.set(bytes, offset, length);
    +        }
           } else if (WritableComparator.compareBytes(minimum.getBytes(), 0,
               minimum.getLength(), bytes, offset, length) > 0) {
    -        minimum = new Text();
    -        minimum.set(bytes, offset, length);
    +        if(length > MAX_BYTES_RECORDED) {
    +          minimum = truncateLowerBound(input);
    +          isLowerBoundSet = true;
    +        } else {
    +          minimum = new Text();
    +          minimum.set(bytes, offset, length);
    --- End diff --
    
    Clear  isLowerBoundSet.


---

Mime
View raw message