Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4A478200BF6 for ; Tue, 10 Jan 2017 16:37:51 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 48E96160B29; Tue, 10 Jan 2017 15:37:51 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2E18B160B3D for ; Tue, 10 Jan 2017 16:37:49 +0100 (CET) Received: (qmail 72861 invoked by uid 500); 10 Jan 2017 15:37:48 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 72766 invoked by uid 99); 10 Jan 2017 15:37:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jan 2017 15:37:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E6A05DFB3D; Tue, 10 Jan 2017 15:37:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mwalch@apache.org To: commits@accumulo.apache.org Date: Tue, 10 Jan 2017 15:37:48 -0000 Message-Id: <69532c310cb8407ba5b28e4b7c622803@git.apache.org> In-Reply-To: <62bf9ef6da464051ae816dc49685a3c5@git.apache.org> References: <62bf9ef6da464051ae816dc49685a3c5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/9] accumulo git commit: ACCUMULO-4510 Removing random walk test code archived-at: Tue, 10 Jan 2017 15:37:51 -0000 http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java deleted file mode 100644 index 08fcb4f..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.core.client.BatchDeleter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.iterators.user.RegExFilter; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -//a test created to test the batch deleter -public class DeleteSomeDocs extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - // delete documents that where the document id matches a given pattern from doc and index table - // using the batch deleter - - Random rand = (Random) state.get("rand"); - String indexTableName = (String) state.get("indexTableName"); - String dataTableName = (String) state.get("docTableName"); - - ArrayList patterns = new ArrayList<>(); - - for (Object key : props.keySet()) - if (key instanceof String && ((String) key).startsWith("pattern")) - patterns.add(props.getProperty((String) key)); - - String pattern = patterns.get(rand.nextInt(patterns.size())); - BatchWriterConfig bwc = new BatchWriterConfig(); - BatchDeleter ibd = env.getConnector().createBatchDeleter(indexTableName, Authorizations.EMPTY, 8, bwc); - ibd.setRanges(Collections.singletonList(new Range())); - - IteratorSetting iterSettings = new IteratorSetting(100, RegExFilter.class); - RegExFilter.setRegexs(iterSettings, null, null, pattern, null, false); - - ibd.addScanIterator(iterSettings); - - ibd.delete(); - - ibd.close(); - - BatchDeleter dbd = env.getConnector().createBatchDeleter(dataTableName, Authorizations.EMPTY, 8, bwc); - dbd.setRanges(Collections.singletonList(new Range())); - - iterSettings = new IteratorSetting(100, RegExFilter.class); - RegExFilter.setRegexs(iterSettings, pattern, null, null, null, false); - - dbd.addScanIterator(iterSettings); - - dbd.delete(); - - dbd.close(); - - log.debug("Deleted documents w/ id matching '" + pattern + "'"); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java deleted file mode 100644 index 9de0240..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.ArrayList; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; -import org.apache.hadoop.io.Text; - -/** - * Delete all documents containing a particular word. - * - */ - -public class DeleteWord extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - String docTableName = (String) state.get("docTableName"); - int numPartitions = (Integer) state.get("numPartitions"); - Random rand = (Random) state.get("rand"); - - String wordToDelete = Insert.generateRandomWord(rand); - - // use index to find all documents containing word - Scanner scanner = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY); - scanner.fetchColumnFamily(new Text(wordToDelete)); - - ArrayList documentsToDelete = new ArrayList<>(); - - for (Entry entry : scanner) - documentsToDelete.add(new Range(entry.getKey().getColumnQualifier())); - - if (documentsToDelete.size() > 0) { - // use a batch scanner to fetch all documents - BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, Authorizations.EMPTY, 8); - bscanner.setRanges(documentsToDelete); - - BatchWriter ibw = env.getMultiTableBatchWriter().getBatchWriter(indexTableName); - BatchWriter dbw = env.getMultiTableBatchWriter().getBatchWriter(docTableName); - - int count = 0; - - for (Entry entry : bscanner) { - String docID = entry.getKey().getRow().toString(); - String doc = entry.getValue().toString(); - - Insert.unindexDocument(ibw, doc, docID, numPartitions); - - Mutation m = new Mutation(docID); - m.putDelete("doc", ""); - - dbw.addMutation(m); - count++; - } - - bscanner.close(); - - env.getMultiTableBatchWriter().flush(); - - if (count != documentsToDelete.size()) { - throw new Exception("Batch scanner did not return expected number of docs " + count + " " + documentsToDelete.size()); - } - } - - log.debug("Deleted " + documentsToDelete.size() + " documents containing " + wordToDelete); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java deleted file mode 100644 index 5b57ace..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.Properties; - -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; - -/** - * - */ -public class ExportIndex extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - - String indexTableName = (String) state.get("indexTableName"); - String tmpIndexTableName = indexTableName + "_tmp"; - - String exportDir = "/tmp/shard_export/" + indexTableName; - String copyDir = "/tmp/shard_export/" + tmpIndexTableName; - - FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); - - fs.delete(new Path("/tmp/shard_export/" + indexTableName), true); - fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true); - - // disable spits, so that splits can be compared later w/o worrying one table splitting and the other not - env.getConnector().tableOperations().setProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "20G"); - - long t1 = System.currentTimeMillis(); - - env.getConnector().tableOperations().flush(indexTableName, null, null, true); - env.getConnector().tableOperations().offline(indexTableName); - - long t2 = System.currentTimeMillis(); - - env.getConnector().tableOperations().exportTable(indexTableName, exportDir); - - long t3 = System.currentTimeMillis(); - - // copy files - BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(new Path(exportDir, "distcp.txt")), UTF_8)); - String file = null; - while ((file = reader.readLine()) != null) { - Path src = new Path(file); - Path dest = new Path(new Path(copyDir), src.getName()); - FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance()); - } - - reader.close(); - - long t4 = System.currentTimeMillis(); - - env.getConnector().tableOperations().online(indexTableName); - env.getConnector().tableOperations().importTable(tmpIndexTableName, copyDir); - - long t5 = System.currentTimeMillis(); - - fs.delete(new Path(exportDir), true); - fs.delete(new Path(copyDir), true); - - HashSet splits1 = new HashSet<>(env.getConnector().tableOperations().listSplits(indexTableName)); - HashSet splits2 = new HashSet<>(env.getConnector().tableOperations().listSplits(tmpIndexTableName)); - - if (!splits1.equals(splits2)) - throw new Exception("Splits not equals " + indexTableName + " " + tmpIndexTableName); - - HashMap props1 = new HashMap<>(); - for (Entry entry : env.getConnector().tableOperations().getProperties(indexTableName)) - props1.put(entry.getKey(), entry.getValue()); - - HashMap props2 = new HashMap<>(); - for (Entry entry : env.getConnector().tableOperations().getProperties(tmpIndexTableName)) - props2.put(entry.getKey(), entry.getValue()); - - if (!props1.equals(props2)) - throw new Exception("Props not equals " + indexTableName + " " + tmpIndexTableName); - - // unset the split threshold - env.getConnector().tableOperations().removeProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey()); - env.getConnector().tableOperations().removeProperty(tmpIndexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey()); - - log.debug("Imported " + tmpIndexTableName + " from " + indexTableName + " flush: " + (t2 - t1) + "ms export: " + (t3 - t2) + "ms copy:" + (t4 - t3) - + "ms import:" + (t5 - t4) + "ms"); - - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java deleted file mode 100644 index 39d32c4..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class Flush extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - String dataTableName = (String) state.get("docTableName"); - Random rand = (Random) state.get("rand"); - - String table; - - if (rand.nextDouble() < .5) - table = indexTableName; - else - table = dataTableName; - - env.getConnector().tableOperations().flush(table, null, null, true); - log.debug("Flushed " + table); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java deleted file mode 100644 index 7409ea7..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.user.IntersectingIterator; -import org.apache.accumulo.core.iterators.user.RegExFilter; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; -import org.apache.hadoop.io.Text; - -public class Grep extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - // pick a few randoms words... grep for those words and search the index - // ensure both return the same set of documents - - String indexTableName = (String) state.get("indexTableName"); - String dataTableName = (String) state.get("docTableName"); - Random rand = (Random) state.get("rand"); - - Text words[] = new Text[rand.nextInt(4) + 2]; - - for (int i = 0; i < words.length; i++) { - words[i] = new Text(Insert.generateRandomWord(rand)); - } - - BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 16); - IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class.getName()); - IntersectingIterator.setColumnFamilies(ii, words); - bs.addScanIterator(ii); - bs.setRanges(Collections.singleton(new Range())); - - HashSet documentsFoundInIndex = new HashSet<>(); - - for (Entry entry2 : bs) { - documentsFoundInIndex.add(entry2.getKey().getColumnQualifier()); - } - - bs.close(); - - bs = env.getConnector().createBatchScanner(dataTableName, Authorizations.EMPTY, 16); - - for (int i = 0; i < words.length; i++) { - IteratorSetting more = new IteratorSetting(20 + i, "ii" + i, RegExFilter.class); - RegExFilter.setRegexs(more, null, null, null, "(^|(.*\\s))" + words[i] + "($|(\\s.*))", false); - bs.addScanIterator(more); - } - - bs.setRanges(Collections.singleton(new Range())); - - HashSet documentsFoundByGrep = new HashSet<>(); - - for (Entry entry2 : bs) { - documentsFoundByGrep.add(entry2.getKey().getRow()); - } - - bs.close(); - - if (!documentsFoundInIndex.equals(documentsFoundByGrep)) { - throw new Exception("Set of documents found not equal for words " + Arrays.asList(words).toString() + " " + documentsFoundInIndex + " " - + documentsFoundByGrep); - } - - log.debug("Grep and index agree " + Arrays.asList(words).toString() + " " + documentsFoundInIndex.size()); - - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java deleted file mode 100644 index 8482a59..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.HashSet; -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class Insert extends Test { - - static final int NUM_WORDS = 100000; - static final int MIN_WORDS_PER_DOC = 10; - static final int MAX_WORDS_PER_DOC = 3000; - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - String dataTableName = (String) state.get("docTableName"); - int numPartitions = (Integer) state.get("numPartitions"); - Random rand = (Random) state.get("rand"); - long nextDocID = (Long) state.get("nextDocID"); - - BatchWriter dataWriter = env.getMultiTableBatchWriter().getBatchWriter(dataTableName); - BatchWriter indexWriter = env.getMultiTableBatchWriter().getBatchWriter(indexTableName); - - String docID = insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand); - - log.debug("Inserted document " + docID); - - state.set("nextDocID", Long.valueOf(nextDocID)); - } - - static String insertRandomDocument(long did, BatchWriter dataWriter, BatchWriter indexWriter, String indexTableName, String dataTableName, int numPartitions, - Random rand) throws TableNotFoundException, Exception, AccumuloException, AccumuloSecurityException { - String doc = createDocument(rand); - - String docID = new StringBuilder(String.format("%016x", did)).reverse().toString(); - - saveDocument(dataWriter, docID, doc); - indexDocument(indexWriter, doc, docID, numPartitions); - - return docID; - } - - static void saveDocument(BatchWriter bw, String docID, String doc) throws Exception { - - Mutation m = new Mutation(docID); - m.put("doc", "", doc); - - bw.addMutation(m); - } - - static String createDocument(Random rand) { - StringBuilder sb = new StringBuilder(); - - int numWords = rand.nextInt(MAX_WORDS_PER_DOC - MIN_WORDS_PER_DOC) + MIN_WORDS_PER_DOC; - - for (int i = 0; i < numWords; i++) { - String word = generateRandomWord(rand); - - if (i > 0) - sb.append(" "); - - sb.append(word); - } - - return sb.toString(); - } - - static String generateRandomWord(Random rand) { - return Integer.toString(rand.nextInt(NUM_WORDS), Character.MAX_RADIX); - } - - static String genPartition(int partition) { - return String.format("%06x", Math.abs(partition)); - } - - static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception { - indexDocument(bw, doc, docId, numPartitions, false); - } - - static void unindexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception { - indexDocument(bw, doc, docId, numPartitions, true); - } - - static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions, boolean delete) throws Exception { - - String[] tokens = doc.split("\\W+"); - - String partition = genPartition(doc.hashCode() % numPartitions); - - Mutation m = new Mutation(partition); - - HashSet tokensSeen = new HashSet<>(); - - for (String token : tokens) { - token = token.toLowerCase(); - - if (!tokensSeen.contains(token)) { - tokensSeen.add(token); - if (delete) - m.putDelete(token, docId); - else - m.put(token, docId, new Value(new byte[0])); - } - } - - if (m.size() > 0) - bw.addMutation(m); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java deleted file mode 100644 index 910e64c..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Collection; -import java.util.Properties; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; -import org.apache.hadoop.io.Text; - -public class Merge extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - - Collection splits = env.getConnector().tableOperations().listSplits(indexTableName); - SortedSet splitSet = new TreeSet<>(splits); - log.debug("merging " + indexTableName); - env.getConnector().tableOperations().merge(indexTableName, null, null); - org.apache.accumulo.core.util.Merge merge = new org.apache.accumulo.core.util.Merge(); - merge.mergomatic(env.getConnector(), indexTableName, null, null, 256 * 1024 * 1024, true); - splits = env.getConnector().tableOperations().listSplits(indexTableName); - if (splits.size() > splitSet.size()) { - // throw an excpetion so that test will die an no further changes to table will occur... - // this way table is left as is for debugging. - throw new Exception("There are more tablets after a merge: " + splits.size() + " was " + splitSet.size()); - } - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java deleted file mode 100644 index 5aed6e3..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class Reindex extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - String tmpIndexTableName = indexTableName + "_tmp"; - String docTableName = (String) state.get("docTableName"); - int numPartitions = (Integer) state.get("numPartitions"); - - Random rand = (Random) state.get("rand"); - - ShardFixture.createIndexTable(this.log, state, env, "_tmp", rand); - - Scanner scanner = env.getConnector().createScanner(docTableName, Authorizations.EMPTY); - BatchWriter tbw = env.getConnector().createBatchWriter(tmpIndexTableName, new BatchWriterConfig()); - - int count = 0; - - for (Entry entry : scanner) { - String docID = entry.getKey().getRow().toString(); - String doc = entry.getValue().toString(); - - Insert.indexDocument(tbw, doc, docID, numPartitions); - - count++; - } - - tbw.close(); - - log.debug("Reindexed " + count + " documents into " + tmpIndexTableName); - - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java deleted file mode 100644 index bccb280..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Random; - -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.user.IntersectingIterator; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; -import org.apache.hadoop.io.Text; - -public class Search extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - String dataTableName = (String) state.get("docTableName"); - - Random rand = (Random) state.get("rand"); - - Entry entry = findRandomDocument(state, env, dataTableName, rand); - if (entry == null) - return; - - Text docID = entry.getKey().getRow(); - String doc = entry.getValue().toString(); - - String[] tokens = doc.split("\\W+"); - int numSearchTerms = rand.nextInt(6); - if (numSearchTerms < 2) - numSearchTerms = 2; - - HashSet searchTerms = new HashSet<>(); - while (searchTerms.size() < numSearchTerms) - searchTerms.add(tokens[rand.nextInt(tokens.length)]); - - Text columns[] = new Text[searchTerms.size()]; - int index = 0; - for (String term : searchTerms) { - columns[index++] = new Text(term); - } - - log.debug("Looking up terms " + searchTerms + " expect to find " + docID); - - BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 10); - IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class); - IntersectingIterator.setColumnFamilies(ii, columns); - bs.addScanIterator(ii); - bs.setRanges(Collections.singleton(new Range())); - - boolean sawDocID = false; - - for (Entry entry2 : bs) { - if (entry2.getKey().getColumnQualifier().equals(docID)) { - sawDocID = true; - break; - } - } - - bs.close(); - - if (!sawDocID) - throw new Exception("Did not see doc " + docID + " in index. terms:" + searchTerms + " " + indexTableName + " " + dataTableName); - } - - static Entry findRandomDocument(State state, Environment env, String dataTableName, Random rand) throws Exception { - Scanner scanner = env.getConnector().createScanner(dataTableName, Authorizations.EMPTY); - scanner.setBatchSize(1); - scanner.setRange(new Range(Integer.toString(rand.nextInt(0xfffffff), 16), null)); - - Iterator> iter = scanner.iterator(); - if (!iter.hasNext()) - return null; - - return iter.next(); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java deleted file mode 100644 index 99e3a61..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.net.InetAddress; -import java.util.Random; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.MultiTableBatchWriter; -import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.Fixture; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.hadoop.io.Text; -import org.apache.log4j.Logger; - -public class ShardFixture extends Fixture { - - static SortedSet genSplits(long max, int numTablets, String format) { - - int numSplits = numTablets - 1; - long distance = max / numTablets; - long split = distance; - - TreeSet splits = new TreeSet<>(); - - for (int i = 0; i < numSplits; i++) { - splits.add(new Text(String.format(format, split))); - split += distance; - } - - return splits; - } - - static void createIndexTable(Logger log, State state, Environment env, String suffix, Random rand) throws Exception { - Connector conn = env.getConnector(); - String name = (String) state.get("indexTableName") + suffix; - int numPartitions = (Integer) state.get("numPartitions"); - boolean enableCache = (Boolean) state.get("cacheIndex"); - conn.tableOperations().create(name); - - String tableId = conn.tableOperations().tableIdMap().get(name); - log.info("Created index table " + name + "(id:" + tableId + ")"); - - SortedSet splits = genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x"); - conn.tableOperations().addSplits(name, splits); - - log.info("Added " + splits.size() + " splits to " + name); - - if (enableCache) { - conn.tableOperations().setProperty(name, Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true"); - conn.tableOperations().setProperty(name, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true"); - - log.info("Enabled caching for table " + name); - } - } - - @Override - public void setUp(State state, Environment env) throws Exception { - String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_"); - String pid = env.getPid(); - - Random rand = new Random(); - - int numPartitions = rand.nextInt(90) + 10; - - state.set("indexTableName", String.format("ST_index_%s_%s_%d", hostname, pid, System.currentTimeMillis())); - state.set("docTableName", String.format("ST_docs_%s_%s_%d", hostname, pid, System.currentTimeMillis())); - state.set("numPartitions", Integer.valueOf(numPartitions)); - state.set("cacheIndex", rand.nextDouble() < .5); - state.set("rand", rand); - state.set("nextDocID", Long.valueOf(0)); - - Connector conn = env.getConnector(); - - createIndexTable(this.log, state, env, "", rand); - - String docTableName = (String) state.get("docTableName"); - conn.tableOperations().create(docTableName); - - String tableId = conn.tableOperations().tableIdMap().get(docTableName); - log.info("Created doc table " + docTableName + " (id:" + tableId + ")"); - - SortedSet splits = genSplits(0xff, rand.nextInt(32) + 1, "%02x"); - conn.tableOperations().addSplits(docTableName, splits); - - log.info("Added " + splits.size() + " splits to " + docTableName); - - if (rand.nextDouble() < .5) { - conn.tableOperations().setProperty((String) state.get("docTableName"), Property.TABLE_BLOOM_ENABLED.getKey(), "true"); - log.info("Enabled bloom filters for table " + (String) state.get("docTableName")); - } - } - - @Override - public void tearDown(State state, Environment env) throws Exception { - // We have resources we need to clean up - if (env.isMultiTableBatchWriterInitialized()) { - MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter(); - try { - mtbw.close(); - } catch (MutationsRejectedException e) { - log.error("Ignoring mutations that weren't flushed", e); - } - - // Reset the MTBW on the state to null - env.resetMultiTableBatchWriter(); - } - - Connector conn = env.getConnector(); - - log.info("Deleting index and doc tables"); - - conn.tableOperations().delete((String) state.get("indexTableName")); - conn.tableOperations().delete((String) state.get("docTableName")); - - log.debug("Exiting shard test"); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java deleted file mode 100644 index cdeccb3..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Collection; - -import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat; -import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.util.Tool; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SortTool extends Configured implements Tool { - protected final Logger log = LoggerFactory.getLogger(this.getClass()); - private String outputDir; - private String seqFile; - private String splitFile; - private Collection splits; - - public SortTool(String seqFile, String outputDir, String splitFile, Collection splits) { - this.outputDir = outputDir; - this.seqFile = seqFile; - this.splitFile = splitFile; - this.splits = splits; - } - - @Override - public int run(String[] args) throws Exception { - Job job = Job.getInstance(getConf(), this.getClass().getSimpleName()); - job.setJarByClass(this.getClass()); - - if (job.getJar() == null) { - log.error("M/R requires a jar file! Run mvn package."); - return 1; - } - - job.setInputFormatClass(SequenceFileInputFormat.class); - SequenceFileInputFormat.setInputPaths(job, seqFile); - - job.setPartitionerClass(KeyRangePartitioner.class); - KeyRangePartitioner.setSplitFile(job, splitFile); - - job.setMapOutputKeyClass(Key.class); - job.setMapOutputValueClass(Value.class); - - job.setNumReduceTasks(splits.size() + 1); - - job.setOutputFormatClass(AccumuloFileOutputFormat.class); - AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir)); - - job.waitForCompletion(true); - return job.isSuccessful() ? 0 : 1; - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java deleted file mode 100644 index 0420fb8..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Properties; -import java.util.Random; -import java.util.SortedSet; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; -import org.apache.hadoop.io.Text; - -public class Split extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - String indexTableName = (String) state.get("indexTableName"); - int numPartitions = (Integer) state.get("numPartitions"); - Random rand = (Random) state.get("rand"); - - SortedSet splitSet = ShardFixture.genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x"); - log.debug("adding splits " + indexTableName); - env.getConnector().tableOperations().addSplits(indexTableName, splitSet); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java deleted file mode 100644 index 70440d8..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.shard; - -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Properties; - -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.PartialKey; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class VerifyIndex extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception { - - String indexTableName = (String) state.get("indexTableName"); - String tmpIndexTableName = indexTableName + "_tmp"; - - // scan new and old index and verify identical - Scanner indexScanner1 = env.getConnector().createScanner(tmpIndexTableName, Authorizations.EMPTY); - Scanner indexScanner2 = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY); - - Iterator> iter = indexScanner2.iterator(); - - int count = 0; - - for (Entry entry : indexScanner1) { - if (!iter.hasNext()) - throw new Exception("index rebuild mismatch " + entry.getKey() + " " + indexTableName); - - Key key1 = entry.getKey(); - Key key2 = iter.next().getKey(); - - if (!key1.equals(key2, PartialKey.ROW_COLFAM_COLQUAL)) - throw new Exception("index rebuild mismatch " + key1 + " " + key2 + " " + indexTableName + " " + tmpIndexTableName); - count++; - if (count % 1000 == 0) - makingProgress(); - } - - if (iter.hasNext()) - throw new Exception("index rebuild mismatch " + iter.next().getKey() + " " + tmpIndexTableName); - - log.debug("Verified " + count + " index entries "); - - env.getConnector().tableOperations().delete(indexTableName); - env.getConnector().tableOperations().rename(tmpIndexTableName, indexTableName); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java deleted file mode 100644 index 2e14de7..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.accumulo.test.randomwalk.unit; - -import java.util.Properties; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class CreateTable extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception {} -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java deleted file mode 100644 index c0b7e65..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.unit; - -import java.util.Properties; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class DeleteTable extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception {} -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java deleted file mode 100644 index 40f3555..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.unit; - -import java.util.Properties; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class Ingest extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception {} -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java deleted file mode 100644 index 5273ea8..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.unit; - -import java.util.Properties; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class Scan extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception {} -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java deleted file mode 100644 index c41d633..0000000 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk.unit; - -import java.util.Properties; - -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.State; -import org.apache.accumulo.test.randomwalk.Test; - -public class Verify extends Test { - - @Override - public void visit(State state, Environment env, Properties props) throws Exception {} -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java deleted file mode 100644 index 80bc69d..0000000 --- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.replication; - -import static org.apache.accumulo.core.conf.Property.TSERV_ARCHIVE_WALOGS; -import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE; - -import java.util.Properties; - -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.accumulo.test.functional.ConfigurableMacBase; -import org.apache.accumulo.test.randomwalk.Environment; -import org.apache.accumulo.test.randomwalk.concurrent.Replication; -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -public class ReplicationRandomWalkIT extends ConfigurableMacBase { - - @Override - protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - cfg.setProperty(TSERV_ARCHIVE_WALOGS, "false"); - cfg.setProperty(TSERV_WALOG_MAX_SIZE, "1M"); - cfg.setNumTservers(1); - } - - @Test(timeout = 5 * 60 * 1000) - public void runReplicationRandomWalkStep() throws Exception { - Replication r = new Replication(); - - Environment env = new Environment(new Properties()) { - @Override - public String getUserName() { - return "root"; - } - - @Override - public String getPassword() { - return ROOT_PASSWORD; - } - - @Override - public Connector getConnector() throws AccumuloException, AccumuloSecurityException { - return ReplicationRandomWalkIT.this.getConnector(); - } - - }; - r.visit(null, env, null); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/resources/randomwalk/module.xsd ---------------------------------------------------------------------- diff --git a/test/src/main/resources/randomwalk/module.xsd b/test/src/main/resources/randomwalk/module.xsd deleted file mode 100644 index bcdaaae0..0000000 --- a/test/src/main/resources/randomwalk/module.xsd +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/resources/unit/Basic.xml ---------------------------------------------------------------------- diff --git a/test/src/main/resources/unit/Basic.xml b/test/src/main/resources/unit/Basic.xml deleted file mode 100644 index 2dead02..0000000 --- a/test/src/main/resources/unit/Basic.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/resources/unit/Simple.xml ---------------------------------------------------------------------- diff --git a/test/src/main/resources/unit/Simple.xml b/test/src/main/resources/unit/Simple.xml deleted file mode 100644 index cad940e..0000000 --- a/test/src/main/resources/unit/Simple.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java b/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java deleted file mode 100644 index ca003ab..0000000 --- a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.randomwalk; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; - -import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; - -import org.apache.accumulo.test.randomwalk.unit.CreateTable; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -public class FrameworkTest { - - // Need to use fully qualified name here because of conflict with org.apache.accumulo.test.randomwalk.Test - @org.junit.Test - public void testXML() throws SAXException, URISyntaxException, ParserConfigurationException, IOException { - SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema moduleSchema = sf.newSchema(getFile("/randomwalk/module.xsd")); - - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - dbf.setSchema(moduleSchema); - - DocumentBuilder docbuilder = dbf.newDocumentBuilder(); - Document document = docbuilder.parse(getFile("/randomwalk/Basic.xml")); - - assertNotEquals("Parsing randomwalk xml should result in nodes.", 0, document.getChildNodes().getLength()); - } - - private File getFile(String resource) throws URISyntaxException { - return new File(this.getClass().getResource(resource).toURI()); - } - - @org.junit.Test - public void testRWTest() { - Test t1 = new CreateTable(); - assertEquals("org.apache.accumulo.test.randomwalk.unit.CreateTable", t1.toString()); - - Test t2 = new CreateTable(); - assertEquals("CreateTable test nodes were not equal.", t1, t2); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/test/resources/randomwalk/Basic.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/randomwalk/Basic.xml b/test/src/test/resources/randomwalk/Basic.xml deleted file mode 100644 index 2dead02..0000000 --- a/test/src/test/resources/randomwalk/Basic.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/test/resources/randomwalk/Simple.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/randomwalk/Simple.xml b/test/src/test/resources/randomwalk/Simple.xml deleted file mode 100644 index cad940e..0000000 --- a/test/src/test/resources/randomwalk/Simple.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/README.md ---------------------------------------------------------------------- diff --git a/test/system/randomwalk/README.md b/test/system/randomwalk/README.md deleted file mode 100644 index 786a160..0000000 --- a/test/system/randomwalk/README.md +++ /dev/null @@ -1,99 +0,0 @@ - - -Apache Accumulo Random Walk Tests -================================= - -The randomwalk framework needs to be configured for your Accumulo instance by -doing the following steps: - -1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your - `$ACCUMULO_CONF_DIR/accumulo-env.sh`. - -2. Create 'randomwalk.conf' file in the `conf` directory containing settings - needed by walkers to connect to Accumulo. - -3. Create a 'walkers' file in the `conf` directory containing the hostnames of - the machines where you want random walkers to run. - -3. Create a 'logger.xml' file in the `conf` directory from `logger.xml.example`. - -The command below starts random walkers on all machines listed in 'walkers'. -The argument `Image.xml` indicates the module to use (which is located at -`conf/modules/Image.xml`): - -> `$ ./bin/start-all.sh Image.xml` - -All modules must be in `conf/modules` and can be referenced without this prefix. -For example, a module located at `conf/modules/foo/bar.xml` is started as -the following: - -> `$ ./bin/start-all.sh foo/bar.xml` - -This command will load all configuration in the `conf` directory to HDFS and -start identical random walkers on each node. These random walkers will -download the current configuration from HDFS and place them in the `tmp/` -directory. - -Random walkers will drop their logs in the `logs/` directory. If you are running -multiple walkers and want ERROR/WARNs dropped to an NFS-hosted log, please set -`NFS_LOGPATH` to a NFS-mounted directory and uncomment the NFS appender in `logger.xml`. - -You can kill all walkers on the machines listed in the 'walkers' file using -the following command: - -> `$ ./bin/kill-all.sh` - -Module-Specific Configuration ------------------------------ - -The user accounts for walkers that run the Concurrent.xml module must have -password-less SSH access to the entire Accumulo cluster, so that they may run -`$ACCUMULO_HOME/bin/start-all.sh`. Note that this is not the same script as the -one that starts random walkers; it is the script to start up an Accumulo -cluster. You can test that access is in place by running -`$ACCUMULO_HOME/bin/start-all.sh` from the command line of each walker account. - -The above access is also needed for any modules that include Concurrent.xml, -e.g., ShortClean.xml, LongClean.xml. - -Other Useful Commands ---------------------- - -Copies configuration in `conf/` to HDFS: - -> `$ copy-config.sh` - -Copies configuration from HDFS into `tmp/` and starts only one local random walker. - -> `$ start-local.sh All.xml` - -Stops all local random walkers: - -> `$ pkill -f randomwalk.Framework` - -Known Issues ------------- - -If you are running randomwalk tests while exercising Hadoop's high availability -(HA) failover capabilities, you should use Hadoop version 2.1.0 or later. -Failover scenarios are more likely to cause randomwalk test failures under -earlier Hadoop versions. See the following issue reports for more details. - -* [HDFS-4404](https://issues.apache.org/jira/browse/HDFS-4404) -* [HADOOP-9792](https://issues.apache.org/jira/browse/HADOOP-9792) - http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/apocalypse.sh ---------------------------------------------------------------------- diff --git a/test/system/randomwalk/bin/apocalypse.sh b/test/system/randomwalk/bin/apocalypse.sh deleted file mode 100755 index e62b008..0000000 --- a/test/system/randomwalk/bin/apocalypse.sh +++ /dev/null @@ -1,49 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#copied below from hadoop-config.sh -this="$0" -while [[ -h "$this" ]]; do - ls=$(ls -ld "$this") - link=$(expr "$ls" : '.*-> \(.*\)$') - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=$(dirname "$this")/"$link" - fi -done -bin=$(dirname "$this") -script=$(basename "$this") -bin=$(cd "$bin"; pwd) -this="$bin/$script" - -ACCUMULO_HOME=$(dirname "$this")/../../../.. -export ACCUMULO_HOME=$(cd "$ACCUMULO_HOME"; pwd) - -if [[ -f $ACCUMULO_HOME/conf/accumulo-env.sh ]] ; then - . "$ACCUMULO_HOME/conf/accumulo-env.sh" -fi - -if [[ -z $HADOOP_PREFIX ]] ; then - echo "HADOOP_PREFIX is not set. Please make sure it's set globally." - exit 1 -fi - -echo 'killing all accumulo processes' -pssh -h "$ACCUMULO_HOME/conf/tservers" "pkill -9 -f app=[tmg].*org.apache.accumulo.start " < /dev/null -pssh -h "$ACCUMULO_HOME/conf/masters" "pkill -9 -f app=[tmg].*org.apache.accumulo.start " < /dev/null -exit 0 http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/copy-config.sh ---------------------------------------------------------------------- diff --git a/test/system/randomwalk/bin/copy-config.sh b/test/system/randomwalk/bin/copy-config.sh deleted file mode 100755 index 48afc19..0000000 --- a/test/system/randomwalk/bin/copy-config.sh +++ /dev/null @@ -1,38 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -if [[ -z $HADOOP_PREFIX ]] ; then - echo "HADOOP_PREFIX is not set. Please make sure it's set globally." - exit 1 -fi - -if [[ -z $ACCUMULO_HOME ]] ; then - echo "ACCUMULO_HOME is not set. Please make sure it's set globally." - exit 1 -fi - -RW_HOME=$ACCUMULO_HOME/test/system/randomwalk - -cd "$RW_HOME" - -tar czf config.tgz conf -"$HADOOP_PREFIX/bin/hadoop" fs -rmr /randomwalk 2>/dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -mkdir /randomwalk -"$HADOOP_PREFIX/bin/hadoop" fs -put config.tgz /randomwalk -"$HADOOP_PREFIX/bin/hadoop" fs -setrep 3 /randomwalk/config.tgz -rm config.tgz http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/kill-all.sh ---------------------------------------------------------------------- diff --git a/test/system/randomwalk/bin/kill-all.sh b/test/system/randomwalk/bin/kill-all.sh deleted file mode 100755 index e4bc0be..0000000 --- a/test/system/randomwalk/bin/kill-all.sh +++ /dev/null @@ -1,40 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#copied below from hadoop-config.sh -this="$0" -while [[ -h "$this" ]]; do - ls=$(ls -ld "$this") - link=$(expr "$ls" : '.*-> \(.*\)$') - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=$(dirname "$this")/"$link" - fi -done -bin=$(dirname "$this") -script=$(basename "$this") -bin=$(cd "$bin"; pwd) -this="$bin/$script" - -ACCUMULO_HOME=$(dirname "$this")/../../../.. -export ACCUMULO_HOME=$(cd "$ACCUMULO_HOME"; pwd) - -RW_HOME=$ACCUMULO_HOME/test/system/randomwalk - -echo 'killing random walkers' -pssh -h "$RW_HOME/conf/walkers" "pkill -f [r]andomwalk.Framework" < /dev/null http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/kill-local.sh ---------------------------------------------------------------------- diff --git a/test/system/randomwalk/bin/kill-local.sh b/test/system/randomwalk/bin/kill-local.sh deleted file mode 100755 index 2b35437..0000000 --- a/test/system/randomwalk/bin/kill-local.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -pkill -f randomwalk.Framework