accumulo-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (ACCUMULO-2232) Combiners can cause deleted data to come back
Date Tue, 22 Sep 2015 04:40:04 GMT

    [ https://issues.apache.org/jira/browse/ACCUMULO-2232?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14901924#comment-14901924
] 

ASF GitHub Bot commented on ACCUMULO-2232:
------------------------------------------

Github user joshelser commented on a diff in the pull request:

    https://github.com/apache/accumulo/pull/47#discussion_r40052599
  
    --- Diff: core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
---
    @@ -786,4 +816,107 @@ public void testAdds() {
         assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 5), Long.MAX_VALUE);
       }
     
    +  private TreeMap<Key,Value> readAll(SortedKeyValueIterator<Key,Value> combiner)
throws Exception {
    +    TreeMap<Key,Value> ret = new TreeMap<Key,Value>();
    +
    +    combiner.seek(new Range(), EMPTY_COL_FAMS, false);
    +
    +    while (combiner.hasTop()) {
    +      ret.put(new Key(combiner.getTopKey()), new Value(combiner.getTopValue()));
    +      combiner.next();
    +    }
    +
    +    return ret;
    +  }
    +
    +  private void runDeleteHandlingTest(TreeMap<Key,Value> input, TreeMap<Key,Value>
expected, DeleteHandlingAction dha, IteratorEnvironment env)
    +      throws Exception {
    +    runDeleteHandlingTest(input, expected, dha, env, null);
    +  }
    +
    +  private void runDeleteHandlingTest(TreeMap<Key,Value> input, TreeMap<Key,Value>
expected, DeleteHandlingAction dha, IteratorEnvironment env,
    +      String expectedLog) throws Exception {
    +    boolean deepCopy = expected == null;
    +
    +    StringWriter writer = new StringWriter();
    +    WriterAppender appender = new WriterAppender(new PatternLayout("%p, %m%n"), writer);
    +    Logger logger = Logger.getLogger(Combiner.class);
    +    boolean additivity = logger.getAdditivity();
    +    try {
    +      logger.addAppender(appender);
    +      logger.setAdditivity(false);
    +
    +      Combiner ai = new SummingCombiner();
    +
    +      IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    +      SummingCombiner.setEncodingType(is, LongCombiner.StringEncoder.class);
    +      Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    +      if (dha != null) {
    +        Combiner.setDeleteHandlingAction(is, dha);
    +      }
    +
    +      ai.init(new SortedMapIterator(input), is.getOptions(), env);
    +
    +      if (deepCopy)
    +        assertEquals(expected, readAll(ai.deepCopy(env)));
    +      assertEquals(expected, readAll(ai));
    +
    +    } finally {
    +      logger.removeAppender(appender);
    +      logger.setAdditivity(additivity);
    +    }
    +
    +    String logMsgs = writer.toString();
    +    if (expectedLog == null) {
    +      Assert.assertTrue(logMsgs, logMsgs.length() == 0);
    +    } else {
    +      logMsgs = logMsgs.replace('\n', ' ');
    +      Assert.assertTrue(logMsgs, logMsgs.matches(expectedLog));
    +    }
    +  }
    +
    +  @Test
    +  public void testDeleteHandling() throws Exception {
    +    Encoder<Long> encoder = LongCombiner.STRING_ENCODER;
    +
    +    TreeMap<Key,Value> input = new TreeMap<Key,Value>();
    +
    +    IteratorEnvironment paritalMajcIe = new CombinerIteratorEnvironment(IteratorScope.majc,
false);
    +    IteratorEnvironment fullMajcIe = new CombinerIteratorEnvironment(IteratorScope.majc,
true);
    +
    +    // keys that aggregate
    +    nkv(input, 1, 1, 1, 1, false, 4l, encoder);
    +    nkv(input, 1, 1, 1, 2, true, 0l, encoder);
    +    nkv(input, 1, 1, 1, 3, false, 2l, encoder);
    +    nkv(input, 1, 1, 1, 4, false, 9l, encoder);
    +
    +    TreeMap<Key,Value> expected = new TreeMap<Key,Value>();
    +    nkv(expected, 1, 1, 1, 1, false, 4l, encoder);
    +    nkv(expected, 1, 1, 1, 2, true, 0l, encoder);
    +    nkv(expected, 1, 1, 1, 4, false, 11l, encoder);
    +
    +    try {
    +      runDeleteHandlingTest(input, expected, DeleteHandlingAction.THROW_EXCEPTION, paritalMajcIe);
    +      Assert.fail();
    +    } catch (IllegalStateException ise) {
    +      Assert.assertTrue(ise.getMessage().contains("Saw a delete during a partial compaction"));
    --- End diff --
    
    Try to include a meaningful error message if the assertion fails. We should be able to
explain the exact reason the test failed just by looking at surefire output.


> Combiners can cause deleted data to come back
> ---------------------------------------------
>
>                 Key: ACCUMULO-2232
>                 URL: https://issues.apache.org/jira/browse/ACCUMULO-2232
>             Project: Accumulo
>          Issue Type: Bug
>          Components: client, tserver
>            Reporter: John Vines
>
> The case-
> 3 files with-
> * 1 with a key, k, with timestamp 0, value 3
> * 1 with a delete of k with timestamp 1
> * 1 with k with timestamp 2, value 2
> The column of k has a summing combiner set on it. The issue here is that depending on
how the major compactions play out, differing values with result. If all 3 files compact,
the correct value of 2 will result. However, if 1 & 3 compact first, they will aggregate
to 5. And then the delete will fall after the combined value, resulting in the result 5 to
persist.
> First and foremost, this should be documented. I think to remedy this, combiners should
only be used on full MajC, not not full ones. This may necessitate a special flag or a new
combiner that implemented the proper semantics.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message