Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0E2CD19C3F for ; Tue, 12 Apr 2016 13:58:20 +0000 (UTC) Received: (qmail 39417 invoked by uid 500); 12 Apr 2016 13:58:19 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 39210 invoked by uid 500); 12 Apr 2016 13:58:19 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 38203 invoked by uid 99); 12 Apr 2016 13:58:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Apr 2016 13:58:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D640FE0556; Tue, 12 Apr 2016 13:58:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dlmarion@apache.org To: commits@accumulo.apache.org Date: Tue, 12 Apr 2016 13:58:48 -0000 Message-Id: <72d03f8692c748e9a314ef4595e5853a@git.apache.org> In-Reply-To: <28ce56b801f84481baae935b546b14cd@git.apache.org> References: <28ce56b801f84481baae935b546b14cd@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [31/39] accumulo git commit: Merge branch '1.7' Merge branch '1.7' Conflicts: server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/aecfbd5c Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/aecfbd5c Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/aecfbd5c Branch: refs/heads/ACCUMULO-4173 Commit: aecfbd5c720d11a9f83912d996d94ab97f4a2423 Parents: 5231842 eb6a038 Author: Josh Elser Authored: Thu Apr 7 12:33:51 2016 -0400 Committer: Josh Elser Committed: Thu Apr 7 12:33:51 2016 -0400 ---------------------------------------------------------------------- .../java/org/apache/accumulo/tserver/InMemoryMap.java | 2 +- .../main/java/org/apache/accumulo/test/InMemoryMapIT.java | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/aecfbd5c/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java ---------------------------------------------------------------------- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java index 1b02f14,c2bf890..ca4719d --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java @@@ -110,47 -84,38 +110,47 @@@ public class InMemoryMap public static final String TYPE_LOCALITY_GROUP_MAP = "LocalityGroupMap"; public static final String TYPE_LOCALITY_GROUP_MAP_NATIVE = "LocalityGroupMap with native"; - public InMemoryMap(boolean useNativeMap, String memDumpDir) { - this(new HashMap>(), useNativeMap, memDumpDir); + private AtomicReference> samplerRef = new AtomicReference<>(null); + + private AccumuloConfiguration config; + + // defer creating sampler until first write. This was done because an empty sample map configured with no sampler will not flush after a user changes sample + // config. + private Sampler getOrCreateSampler() { + Pair pair = samplerRef.get(); + if (pair == null) { + pair = getSampler(config); + if (!samplerRef.compareAndSet(null, pair)) { + pair = samplerRef.get(); + } + } + + return pair.getSecond(); } - public InMemoryMap(Map> lggroups, boolean useNativeMap, String memDumpDir) { - this.memDumpDir = memDumpDir; - this.lggroups = lggroups; + public InMemoryMap(AccumuloConfiguration config) throws LocalityGroupConfigurationError { + + boolean useNativeMap = config.getBoolean(Property.TSERV_NATIVEMAP_ENABLED); + + this.memDumpDir = config.get(Property.TSERV_MEMDUMP_DIR); + this.lggroups = LocalityGroupUtil.getLocalityGroups(config); + + this.config = config; + + SimpleMap allMap; + SimpleMap sampleMap; if (lggroups.size() == 0) { - map = newMap(useNativeMap); + allMap = newMap(useNativeMap); + sampleMap = newMap(useNativeMap); mapType = useNativeMap ? TYPE_NATIVE_MAP_WRAPPER : TYPE_DEFAULT_MAP; } else { - map = new LocalityGroupMap(lggroups, useNativeMap); + allMap = new LocalityGroupMap(lggroups, useNativeMap); + sampleMap = new LocalityGroupMap(lggroups, useNativeMap); - mapType = useNativeMap ? TYPE_LOCALITY_GROUP_MAP : TYPE_LOCALITY_GROUP_MAP_NATIVE; + mapType = useNativeMap ? TYPE_LOCALITY_GROUP_MAP_NATIVE : TYPE_LOCALITY_GROUP_MAP; } - } - /** - * Description of the type of SimpleMap that is created. - *

- * If no locality groups are present, the SimpleMap is either TYPE_DEFAULT_MAP or TYPE_NATIVE_MAP_WRAPPER. If there is one more locality groups, then the - * InMemoryMap has an array for simple maps that either contain either TYPE_LOCALITY_GROUP_MAP which contains DefaultMaps or TYPE_LOCALITY_GROUP_MAP_NATIVE - * which contains NativeMapWrappers. - * - * @return String that describes the Map type - */ - public String getMapType() { - return mapType; - } - - public InMemoryMap(AccumuloConfiguration config) throws LocalityGroupConfigurationError { - this(LocalityGroupUtil.getLocalityGroups(config), config.getBoolean(Property.TSERV_NATIVEMAP_ENABLED), config.get(Property.TSERV_MEMDUMP_DIR)); + map = new SampleMap(allMap, sampleMap); } private static SimpleMap newMap(boolean useNativeMap) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/aecfbd5c/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java index cdb09d9,0000000..299a13d mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java +++ b/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java @@@ -1,361 -1,0 +1,367 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import com.google.common.collect.ImmutableSet; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.ArrayByteSequence; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.test.functional.NativeMapIT; +import org.apache.accumulo.tserver.InMemoryMap; +import org.apache.accumulo.tserver.MemKey; +import org.apache.accumulo.tserver.NativeMap; +import org.apache.hadoop.io.Text; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration Test for https://issues.apache.org/jira/browse/ACCUMULO-4148 + *

+ * User had problem writing one Mutation with multiple KV pairs that had the same key. Doing so should write out all pairs in all mutations with a unique id. In + * typical operation, you would only see the last one when scanning. User had a combiner on the table, and they noticed that when using InMemoryMap with + * NativeMapWrapper, only the last KV pair was ever written. When InMemoryMap used DefaultMap, all KV pairs were added and the behavior worked as expected. + * + * This IT inserts a variety of Mutations with and without the same KV pairs and then inspects result of InMemoryMap mutate, looking for unique id stored with + * each key. This unique id, shown as mc= in the MemKey toString, was originally used for scan Isolation. Writing the same key multiple times in the same + * mutation is a secondary use case, discussed in https://issues.apache.org/jira/browse/ACCUMULO-227. In addition to NativeMapWrapper and DefaultMap, + * LocalityGroupMap was add in https://issues.apache.org/jira/browse/ACCUMULO-112. + * + * This test has to be an IT in accumulo-test, because libaccumulo is built in 'integration-test' phase of accumulo-native, which currently runs right before + * accumulo-test. The tests for DefaultMap could move to a unit test in tserver, but they are here for convenience of viewing both at the same time. + */ +public class InMemoryMapIT { + + private static final Logger log = LoggerFactory.getLogger(InMemoryMapIT.class); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); + + @BeforeClass + public static void ensureNativeLibrary() throws FileNotFoundException { + File nativeMapLocation = NativeMapIT.nativeMapLocation(); + log.debug("Native map location " + nativeMapLocation); + NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation)); + if (!NativeMap.isLoaded()) { + fail("Missing the native library from " + nativeMapLocation.getAbsolutePath() + "\nYou need to build the libaccumulo binary first. " + + "\nTry running 'mvn clean install -Dit.test=InMemoryMapIT -Dtest=foo -DfailIfNoTests=false -Dfindbugs.skip -Dcheckstyle.skip'"); + // afterwards, you can run the following + // mvn clean verify -Dit.test=InMemoryMapIT -Dtest=foo -DfailIfNoTests=false -Dfindbugs.skip -Dcheckstyle.skip -pl :accumulo-test + } + log.debug("Native map loaded"); + + } + + @Test + public void testOneMutationOneKey() { + Mutation m = new Mutation("a"); + m.put(new Text("1cf"), new Text("1cq"), new Value("vala".getBytes())); + + assertEquivalentMutate(m); + } + + @Test + public void testOneMutationManyKeys() throws IOException { + Mutation m = new Mutation("a"); + for (int i = 1; i < 6; i++) { + m.put(new Text("2cf" + i), new Text("2cq" + i), new Value(Integer.toString(i).getBytes())); + } + + assertEquivalentMutate(m); + } + + @Test + public void testOneMutationManySameKeys() { + Mutation m = new Mutation("a"); + for (int i = 1; i <= 5; i++) { + // same keys + m.put(new Text("3cf"), new Text("3cq"), new Value(Integer.toString(i).getBytes())); + } + + assertEquivalentMutate(m); + } + + @Test + public void testMultipleMutationsOneKey() { + Mutation m1 = new Mutation("a"); + m1.put(new Text("4cf"), new Text("4cq"), new Value("vala".getBytes())); + Mutation m2 = new Mutation("b"); + m2.put(new Text("4cf"), new Text("4cq"), new Value("vala".getBytes())); + + assertEquivalentMutate(Arrays.asList(m1, m2)); + } + + @Test + public void testMultipleMutationsSameOneKey() { + Mutation m1 = new Mutation("a"); + m1.put(new Text("5cf"), new Text("5cq"), new Value("vala".getBytes())); + Mutation m2 = new Mutation("a"); + m2.put(new Text("5cf"), new Text("5cq"), new Value("vala".getBytes())); + + assertEquivalentMutate(Arrays.asList(m1, m2)); + } + + @Test + public void testMutlipleMutationsMultipleKeys() { + Mutation m1 = new Mutation("a"); + for (int i = 1; i < 6; i++) { + m1.put(new Text("6cf" + i), new Text("6cq" + i), new Value(Integer.toString(i).getBytes())); + } + Mutation m2 = new Mutation("b"); + for (int i = 1; i < 3; i++) { + m2.put(new Text("6cf" + i), new Text("6cq" + i), new Value(Integer.toString(i).getBytes())); + } + + assertEquivalentMutate(Arrays.asList(m1, m2)); + } + + @Test + public void testMultipleMutationsMultipleSameKeys() { + Mutation m1 = new Mutation("a"); + for (int i = 1; i < 3; i++) { + m1.put(new Text("7cf"), new Text("7cq"), new Value(Integer.toString(i).getBytes())); + } + Mutation m2 = new Mutation("a"); + for (int i = 1; i < 4; i++) { + m2.put(new Text("7cf"), new Text("7cq"), new Value(Integer.toString(i).getBytes())); + } + + assertEquivalentMutate(Arrays.asList(m1, m2)); + } + + @Test + public void testMultipleMutationsMultipleKeysSomeSame() { + Mutation m1 = new Mutation("a"); + for (int i = 1; i < 2; i++) { + m1.put(new Text("8cf"), new Text("8cq"), new Value(Integer.toString(i).getBytes())); + } + for (int i = 1; i < 3; i++) { + m1.put(new Text("8cf" + i), new Text("8cq" + i), new Value(Integer.toString(i).getBytes())); + } + for (int i = 1; i < 2; i++) { + m1.put(new Text("8cf" + i), new Text("8cq" + i), new Value(Integer.toString(i).getBytes())); + } + Mutation m2 = new Mutation("a"); + for (int i = 1; i < 3; i++) { + m2.put(new Text("8cf"), new Text("8cq"), new Value(Integer.toString(i).getBytes())); + } + for (int i = 1; i < 4; i++) { + m2.put(new Text("8cf" + i), new Text("8cq" + i), new Value(Integer.toString(i).getBytes())); + } + Mutation m3 = new Mutation("b"); + for (int i = 1; i < 3; i++) { + m3.put(new Text("8cf" + i), new Text("8cq" + i), new Value(Integer.toString(i).getBytes())); + } + + assertEquivalentMutate(Arrays.asList(m1, m2, m3)); + } + + private void assertEquivalentMutate(Mutation m) { + assertEquivalentMutate(Collections.singletonList(m)); + } + + private void assertEquivalentMutate(List mutations) { + InMemoryMap defaultMap = null; + InMemoryMap nativeMapWrapper = null; + InMemoryMap localityGroupMap = null; + InMemoryMap localityGroupMapWithNative = null; + + try { + Map defaultMapConfig = new HashMap<>(); + defaultMapConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false"); + defaultMapConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath()); + defaultMapConfig.put(Property.TABLE_LOCALITY_GROUPS.getKey(), ""); + Map nativeMapConfig = new HashMap<>(); + nativeMapConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "true"); + nativeMapConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath()); + nativeMapConfig.put(Property.TABLE_LOCALITY_GROUPS.getKey(), ""); + Map localityGroupConfig = new HashMap<>(); + localityGroupConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false"); + localityGroupConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath()); + Map localityGroupNativeConfig = new HashMap<>(); - localityGroupNativeConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false"); ++ localityGroupNativeConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "true"); + localityGroupNativeConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath()); + + defaultMap = new InMemoryMap(new ConfigurationCopy(defaultMapConfig)); + nativeMapWrapper = new InMemoryMap(new ConfigurationCopy(nativeMapConfig)); + localityGroupMap = new InMemoryMap(updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupConfig))); + localityGroupMapWithNative = new InMemoryMap(updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupNativeConfig))); + } catch (Exception e) { + log.error("Error getting new InMemoryMap ", e); + fail(e.getMessage()); + } + ++ // ensure the maps are correct type ++ assertEquals("Not a DefaultMap", InMemoryMap.TYPE_DEFAULT_MAP, defaultMap.getMapType()); ++ assertEquals("Not a NativeMapWrapper", InMemoryMap.TYPE_NATIVE_MAP_WRAPPER, nativeMapWrapper.getMapType()); ++ assertEquals("Not a LocalityGroupMap", InMemoryMap.TYPE_LOCALITY_GROUP_MAP, localityGroupMap.getMapType()); ++ assertEquals("Not a LocalityGroupMap with native", InMemoryMap.TYPE_LOCALITY_GROUP_MAP_NATIVE, localityGroupMapWithNative.getMapType()); ++ + defaultMap.mutate(mutations); + nativeMapWrapper.mutate(mutations); + localityGroupMap.mutate(mutations); + localityGroupMapWithNative.mutate(mutations); + + // let's use the transitive property to assert all four are equivalent + assertMutatesEquivalent(mutations, defaultMap, nativeMapWrapper); + assertMutatesEquivalent(mutations, defaultMap, localityGroupMap); + assertMutatesEquivalent(mutations, defaultMap, localityGroupMapWithNative); + } + + /** + * Assert that a set of mutations mutate to equivalent map in both of the InMemoryMaps. + *

+ * In this case, equivalent means 2 things. + *

    + *
  • The size of both maps generated is equal to the number of key value pairs in all mutations passed
  • + *
  • The size of the map generated from the first InMemoryMap equals the size of the map generated from the second
  • + *
  • Each key value pair in each mutated map has a unique id (kvCount)
  • + *
+ * + * @param mutations + * List of mutations + * @param imm1 + * InMemoryMap to compare + * @param imm2 + * InMemoryMap to compare + */ + private void assertMutatesEquivalent(List mutations, InMemoryMap imm1, InMemoryMap imm2) { + int mutationKVPairs = countKVPairs(mutations); + + List memKeys1 = getArrayOfMemKeys(imm1); + List memKeys2 = getArrayOfMemKeys(imm2); + + assertEquals("Not all key value pairs included: " + dumpInMemoryMap(imm1, memKeys1), mutationKVPairs, memKeys1.size()); + assertEquals("InMemoryMaps differ in size: " + dumpInMemoryMap(imm1, memKeys1) + "\n" + dumpInMemoryMap(imm2, memKeys2), memKeys1.size(), memKeys2.size()); + assertEquals("InMemoryMap did not have distinct kvCounts " + dumpInMemoryMap(imm1, memKeys1), mutationKVPairs, getUniqKVCount(memKeys1)); + assertEquals("InMemoryMap did not have distinct kvCounts " + dumpInMemoryMap(imm2, memKeys2), mutationKVPairs, getUniqKVCount(memKeys2)); + + } + + private int countKVPairs(List mutations) { + int count = 0; + for (Mutation m : mutations) { + count += m.size(); + } + return count; + } + + private List getArrayOfMemKeys(InMemoryMap imm) { + SortedKeyValueIterator skvi = imm.compactionIterator(); + + List memKeys = new ArrayList(); + try { + skvi.seek(new Range(), new ArrayList(), false); // everything + while (skvi.hasTop()) { + memKeys.add((MemKey) skvi.getTopKey()); + skvi.next(); + } + } catch (IOException ex) { + log.error("Error getting memkeys", ex); + throw new RuntimeException(ex); + } + + return memKeys; + } + + private String dumpInMemoryMap(InMemoryMap map, List memkeys) { + StringBuilder sb = new StringBuilder(); + sb.append("InMemoryMap type "); + sb.append(map.getMapType()); + sb.append("\n"); + + for (MemKey mk : memkeys) { + sb.append(" "); + sb.append(mk.toString()); + sb.append("\n"); + } + + return sb.toString(); + } + + private int getUniqKVCount(List memKeys) { + List kvCounts = new ArrayList(); + for (MemKey m : memKeys) { + kvCounts.add(m.getKVCount()); + } + return ImmutableSet.copyOf(kvCounts).size(); + } + + private ConfigurationCopy updateConfigurationForLocalityGroups(ConfigurationCopy configuration) { + Map> locGroups = getLocalityGroups(); + StringBuilder enabledLGs = new StringBuilder(); + + for (Entry> entry : locGroups.entrySet()) { + if (enabledLGs.length() > 0) { + enabledLGs.append(","); + } + + StringBuilder value = new StringBuilder(); + for (ByteSequence bytes : entry.getValue()) { + if (value.length() > 0) { + value.append(","); + } + value.append(new String(bytes.toArray())); + } + configuration.set("table.group." + entry.getKey(), value.toString()); + enabledLGs.append(entry.getKey()); + } + configuration.set(Property.TABLE_LOCALITY_GROUPS, enabledLGs.toString()); + return configuration; + } + + private Map> getLocalityGroups() { + Map> locgro = new HashMap>(); + locgro.put("a", newCFSet("cf", "cf2")); - locgro.put("a", newCFSet("cf3", "cf4")); ++ locgro.put("b", newCFSet("cf3", "cf4")); + return locgro; + } + + // from InMemoryMapTest + private Set newCFSet(String... cfs) { + HashSet cfSet = new HashSet(); + for (String cf : cfs) { + cfSet.add(new ArrayByteSequence(cf)); + } + return cfSet; + } + +}