accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [4/7] accumulo git commit: ACCUMULO-4511 Removed Accumulo Examples
Date Fri, 09 Dec 2016 17:16:50 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
deleted file mode 100644
index 2c76264..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.dirlist;
-
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.RegExFilter;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Provides utility methods for getting the info for a file, listing the contents of a directory, and performing single wild card searches on file or directory
- * names. See docs/examples/README.dirlist for instructions.
- */
-public class QueryUtil {
-  private Connector conn = null;
-  private String tableName;
-  private Authorizations auths;
-  public static final Text DIR_COLF = new Text("dir");
-  public static final Text FORWARD_PREFIX = new Text("f");
-  public static final Text REVERSE_PREFIX = new Text("r");
-  public static final Text INDEX_COLF = new Text("i");
-  public static final Text COUNTS_COLQ = new Text("counts");
-
-  public QueryUtil(Opts opts) throws AccumuloException, AccumuloSecurityException {
-    conn = opts.getConnector();
-    this.tableName = opts.getTableName();
-    this.auths = opts.auths;
-  }
-
-  /**
-   * Calculates the depth of a path, i.e. the number of forward slashes in the path name.
-   *
-   * @param path
-   *          the full path of a file or directory
-   * @return the depth of the path
-   */
-  public static int getDepth(String path) {
-    int numSlashes = 0;
-    int index = -1;
-    while ((index = path.indexOf("/", index + 1)) >= 0)
-      numSlashes++;
-    return numSlashes;
-  }
-
-  /**
-   * Given a path, construct an accumulo row prepended with the path's depth for the directory table.
-   *
-   * @param path
-   *          the full path of a file or directory
-   * @return the accumulo row associated with this path
-   */
-  public static Text getRow(String path) {
-    Text row = new Text(String.format("%03d", getDepth(path)));
-    row.append(path.getBytes(), 0, path.length());
-    return row;
-  }
-
-  /**
-   * Given a path, construct an accumulo row prepended with the {@link #FORWARD_PREFIX} for the index table.
-   *
-   * @param path
-   *          the full path of a file or directory
-   * @return the accumulo row associated with this path
-   */
-  public static Text getForwardIndex(String path) {
-    String part = path.substring(path.lastIndexOf("/") + 1);
-    if (part.length() == 0)
-      return null;
-    Text row = new Text(FORWARD_PREFIX);
-    row.append(part.getBytes(), 0, part.length());
-    return row;
-  }
-
-  /**
-   * Given a path, construct an accumulo row prepended with the {@link #REVERSE_PREFIX} with the path reversed for the index table.
-   *
-   * @param path
-   *          the full path of a file or directory
-   * @return the accumulo row associated with this path
-   */
-  public static Text getReverseIndex(String path) {
-    String part = path.substring(path.lastIndexOf("/") + 1);
-    if (part.length() == 0)
-      return null;
-    byte[] rev = new byte[part.length()];
-    int i = part.length() - 1;
-    for (byte b : part.getBytes())
-      rev[i--] = b;
-    Text row = new Text(REVERSE_PREFIX);
-    row.append(rev, 0, rev.length);
-    return row;
-  }
-
-  /**
-   * Returns either the {@link #DIR_COLF} or a decoded string version of the colf.
-   *
-   * @param colf
-   *          the column family
-   */
-  public static String getType(Text colf) {
-    if (colf.equals(DIR_COLF))
-      return colf.toString() + ":";
-    return Long.toString(Ingest.encoder.decode(colf.getBytes())) + ":";
-  }
-
-  /**
-   * Scans over the directory table and pulls out stat information about a path.
-   *
-   * @param path
-   *          the full path of a file or directory
-   */
-  public Map<String,String> getData(String path) throws TableNotFoundException {
-    if (path.endsWith("/"))
-      path = path.substring(0, path.length() - 1);
-    Scanner scanner = conn.createScanner(tableName, auths);
-    scanner.setRange(new Range(getRow(path)));
-    Map<String,String> data = new TreeMap<>();
-    for (Entry<Key,Value> e : scanner) {
-      String type = getType(e.getKey().getColumnFamily());
-      data.put("fullname", e.getKey().getRow().toString().substring(3));
-      data.put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
-    }
-    return data;
-  }
-
-  /**
-   * Uses the directory table to list the contents of a directory.
-   *
-   * @param path
-   *          the full path of a directory
-   */
-  public Map<String,Map<String,String>> getDirList(String path) throws TableNotFoundException {
-    if (!path.endsWith("/"))
-      path = path + "/";
-    Map<String,Map<String,String>> fim = new TreeMap<>();
-    Scanner scanner = conn.createScanner(tableName, auths);
-    scanner.setRange(Range.prefix(getRow(path)));
-    for (Entry<Key,Value> e : scanner) {
-      String name = e.getKey().getRow().toString();
-      name = name.substring(name.lastIndexOf("/") + 1);
-      String type = getType(e.getKey().getColumnFamily());
-      if (!fim.containsKey(name)) {
-        fim.put(name, new TreeMap<String,String>());
-        fim.get(name).put("fullname", e.getKey().getRow().toString().substring(3));
-      }
-      fim.get(name).put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
-    }
-    return fim;
-  }
-
-  /**
-   * Scans over the index table for files or directories with a given name.
-   *
-   * @param term
-   *          the name a file or directory to search for
-   */
-  public Iterable<Entry<Key,Value>> exactTermSearch(String term) throws Exception {
-    System.out.println("executing exactTermSearch for " + term);
-    Scanner scanner = conn.createScanner(tableName, auths);
-    scanner.setRange(new Range(getForwardIndex(term)));
-    return scanner;
-  }
-
-  /**
-   * Scans over the index table for files or directories with a given name, prefix, or suffix (indicated by a wildcard '*' at the beginning or end of the term.
-   *
-   * @param exp
-   *          the name a file or directory to search for with an optional wildcard '*' at the beginning or end
-   */
-  public Iterable<Entry<Key,Value>> singleRestrictedWildCardSearch(String exp) throws Exception {
-    if (exp.indexOf("/") >= 0)
-      throw new Exception("this method only works with unqualified names");
-
-    Scanner scanner = conn.createScanner(tableName, auths);
-    if (exp.startsWith("*")) {
-      System.out.println("executing beginning wildcard search for " + exp);
-      exp = exp.substring(1);
-      scanner.setRange(Range.prefix(getReverseIndex(exp)));
-    } else if (exp.endsWith("*")) {
-      System.out.println("executing ending wildcard search for " + exp);
-      exp = exp.substring(0, exp.length() - 1);
-      scanner.setRange(Range.prefix(getForwardIndex(exp)));
-    } else if (exp.indexOf("*") >= 0) {
-      throw new Exception("this method only works for beginning or ending wild cards");
-    } else {
-      return exactTermSearch(exp);
-    }
-    return scanner;
-  }
-
-  /**
-   * Scans over the index table for files or directories with a given name that can contain a single wildcard '*' anywhere in the term.
-   *
-   * @param exp
-   *          the name a file or directory to search for with one optional wildcard '*'
-   */
-  public Iterable<Entry<Key,Value>> singleWildCardSearch(String exp) throws Exception {
-    int starIndex = exp.indexOf("*");
-    if (exp.indexOf("*", starIndex + 1) >= 0)
-      throw new Exception("only one wild card for search");
-
-    if (starIndex < 0) {
-      return exactTermSearch(exp);
-    } else if (starIndex == 0 || starIndex == exp.length() - 1) {
-      return singleRestrictedWildCardSearch(exp);
-    }
-
-    String firstPart = exp.substring(0, starIndex);
-    String lastPart = exp.substring(starIndex + 1);
-    String regexString = ".*/" + exp.replace("*", "[^/]*");
-
-    Scanner scanner = conn.createScanner(tableName, auths);
-    if (firstPart.length() >= lastPart.length()) {
-      System.out.println("executing middle wildcard search for " + regexString + " from entries starting with " + firstPart);
-      scanner.setRange(Range.prefix(getForwardIndex(firstPart)));
-    } else {
-      System.out.println("executing middle wildcard search for " + regexString + " from entries ending with " + lastPart);
-      scanner.setRange(Range.prefix(getReverseIndex(lastPart)));
-    }
-    IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
-    RegExFilter.setRegexs(regex, null, null, regexString, null, false);
-    scanner.addScanIterator(regex);
-    return scanner;
-  }
-
-  public static class Opts extends ClientOnRequiredTable {
-    @Parameter(names = "--path", description = "the directory to list")
-    String path = "/";
-    @Parameter(names = "--search", description = "find a file or directory with the given name")
-    boolean search = false;
-  }
-
-  /**
-   * Lists the contents of a directory using the directory table, or searches for file or directory names (if the -search flag is included).
-   */
-  public static void main(String[] args) throws Exception {
-    Opts opts = new Opts();
-    opts.parseArgs(QueryUtil.class.getName(), args);
-    QueryUtil q = new QueryUtil(opts);
-    if (opts.search) {
-      for (Entry<Key,Value> e : q.singleWildCardSearch(opts.path)) {
-        System.out.println(e.getKey().getColumnQualifier());
-      }
-    } else {
-      for (Entry<String,Map<String,String>> e : q.getDirList(opts.path).entrySet()) {
-        System.out.println(e);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java
deleted file mode 100644
index 2648a48..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.dirlist;
-
-import java.awt.BorderLayout;
-import java.io.IOException;
-import java.util.Enumeration;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import javax.swing.JFrame;
-import javax.swing.JScrollPane;
-import javax.swing.JSplitPane;
-import javax.swing.JTextArea;
-import javax.swing.JTree;
-import javax.swing.event.TreeExpansionEvent;
-import javax.swing.event.TreeExpansionListener;
-import javax.swing.event.TreeSelectionEvent;
-import javax.swing.event.TreeSelectionListener;
-import javax.swing.tree.DefaultMutableTreeNode;
-import javax.swing.tree.DefaultTreeModel;
-import javax.swing.tree.TreePath;
-
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.examples.simple.filedata.FileDataQuery;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Provides a GUI for browsing the file system information stored in Accumulo. See docs/examples/README.dirlist for instructions.
- */
-@SuppressWarnings("serial")
-public class Viewer extends JFrame implements TreeSelectionListener, TreeExpansionListener {
-  private static final Logger log = LoggerFactory.getLogger(Viewer.class);
-
-  JTree tree;
-  DefaultTreeModel treeModel;
-  QueryUtil q;
-  FileDataQuery fdq;
-  String topPath;
-  Map<String,DefaultMutableTreeNode> nodeNameMap;
-  JTextArea text;
-  JTextArea data;
-  JScrollPane dataPane;
-
-  public static class NodeInfo {
-    private String name;
-    private Map<String,String> data;
-
-    public NodeInfo(String name, Map<String,String> data) {
-      this.name = name;
-      this.data = data;
-    }
-
-    public String getName() {
-      return name;
-    }
-
-    public String getFullName() {
-      String fn = data.get("fullname");
-      if (fn == null)
-        return name;
-      return fn;
-    }
-
-    public Map<String,String> getData() {
-      return data;
-    }
-
-    @Override
-    public String toString() {
-      return getName();
-    }
-
-    public String getHash() {
-      for (String k : data.keySet()) {
-        String[] parts = k.split(":");
-        if (parts.length >= 2 && parts[1].equals("md5")) {
-          return data.get(k);
-        }
-      }
-      return null;
-    }
-  }
-
-  public Viewer(Opts opts) throws Exception {
-    super("File Viewer");
-    setSize(1000, 800);
-    setDefaultCloseOperation(EXIT_ON_CLOSE);
-    q = new QueryUtil(opts);
-    fdq = new FileDataQuery(opts.instance, opts.zookeepers, opts.getPrincipal(), opts.getToken(), opts.dataTable, opts.auths);
-    this.topPath = opts.path;
-  }
-
-  public void populate(DefaultMutableTreeNode node) throws TableNotFoundException {
-    String path = ((NodeInfo) node.getUserObject()).getFullName();
-    log.debug("listing " + path);
-    for (Entry<String,Map<String,String>> e : q.getDirList(path).entrySet()) {
-      log.debug("got child for " + node.getUserObject() + ": " + e.getKey());
-      node.add(new DefaultMutableTreeNode(new NodeInfo(e.getKey(), e.getValue())));
-    }
-  }
-
-  public void populateChildren(DefaultMutableTreeNode node) throws TableNotFoundException {
-    @SuppressWarnings("unchecked")
-    Enumeration<DefaultMutableTreeNode> children = node.children();
-    while (children.hasMoreElements()) {
-      populate(children.nextElement());
-    }
-  }
-
-  public void init() throws TableNotFoundException {
-    DefaultMutableTreeNode root = new DefaultMutableTreeNode(new NodeInfo(topPath, q.getData(topPath)));
-    populate(root);
-    populateChildren(root);
-
-    treeModel = new DefaultTreeModel(root);
-    tree = new JTree(treeModel);
-    tree.addTreeExpansionListener(this);
-    tree.addTreeSelectionListener(this);
-    text = new JTextArea(getText(q.getData(topPath)));
-    data = new JTextArea("");
-    JScrollPane treePane = new JScrollPane(tree);
-    JScrollPane textPane = new JScrollPane(text);
-    dataPane = new JScrollPane(data);
-    JSplitPane infoSplitPane = new JSplitPane(JSplitPane.VERTICAL_SPLIT, textPane, dataPane);
-    JSplitPane mainSplitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, treePane, infoSplitPane);
-    mainSplitPane.setDividerLocation(300);
-    infoSplitPane.setDividerLocation(150);
-    getContentPane().add(mainSplitPane, BorderLayout.CENTER);
-  }
-
-  public static String getText(DefaultMutableTreeNode node) {
-    return getText(((NodeInfo) node.getUserObject()).getData());
-  }
-
-  public static String getText(Map<String,String> data) {
-    StringBuilder sb = new StringBuilder();
-    for (String name : data.keySet()) {
-      sb.append(name);
-      sb.append(" : ");
-      sb.append(data.get(name));
-      sb.append('\n');
-    }
-    return sb.toString();
-  }
-
-  @Override
-  public void treeExpanded(TreeExpansionEvent event) {
-    try {
-      populateChildren((DefaultMutableTreeNode) event.getPath().getLastPathComponent());
-    } catch (TableNotFoundException e) {
-      log.error("Could not find table.", e);
-    }
-  }
-
-  @Override
-  public void treeCollapsed(TreeExpansionEvent event) {
-    DefaultMutableTreeNode node = (DefaultMutableTreeNode) event.getPath().getLastPathComponent();
-    @SuppressWarnings("unchecked")
-    Enumeration<DefaultMutableTreeNode> children = node.children();
-    while (children.hasMoreElements()) {
-      DefaultMutableTreeNode child = children.nextElement();
-      log.debug("removing children of " + ((NodeInfo) child.getUserObject()).getFullName());
-      child.removeAllChildren();
-    }
-  }
-
-  @Override
-  public void valueChanged(TreeSelectionEvent e) {
-    TreePath selected = e.getNewLeadSelectionPath();
-    if (selected == null)
-      return;
-    DefaultMutableTreeNode node = (DefaultMutableTreeNode) selected.getLastPathComponent();
-    text.setText(getText(node));
-    try {
-      String hash = ((NodeInfo) node.getUserObject()).getHash();
-      if (hash != null) {
-        data.setText(fdq.getSomeData(hash, 10000));
-      } else {
-        data.setText("");
-      }
-    } catch (IOException e1) {
-      log.error("Could not get data from FileDataQuery.", e1);
-    }
-  }
-
-  static class Opts extends QueryUtil.Opts {
-    @Parameter(names = "--dataTable")
-    String dataTable = "dataTable";
-  }
-
-  public static void main(String[] args) throws Exception {
-    Opts opts = new Opts();
-    opts.parseArgs(Viewer.class.getName(), args);
-
-    Viewer v = new Viewer(opts);
-    v.init();
-    v.setVisible(true);
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java
deleted file mode 100644
index 9e32090..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.cli.MapReduceClientOnRequiredTable;
-import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.SummingArrayCombiner;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * A MapReduce that computes a histogram of byte frequency for each file and stores the histogram alongside the file data. The {@link ChunkInputFormat} is used
- * to read the file data from Accumulo. See docs/examples/README.filedata for instructions.
- */
-public class CharacterHistogram extends Configured implements Tool {
-  public static final String VIS = "vis";
-
-  public static void main(String[] args) throws Exception {
-    System.exit(ToolRunner.run(new Configuration(), new CharacterHistogram(), args));
-  }
-
-  public static class HistMapper extends Mapper<List<Entry<Key,Value>>,InputStream,Text,Mutation> {
-    private ColumnVisibility cv;
-
-    @Override
-    public void map(List<Entry<Key,Value>> k, InputStream v, Context context) throws IOException, InterruptedException {
-      Long[] hist = new Long[256];
-      for (int i = 0; i < hist.length; i++)
-        hist[i] = 0l;
-      int b = v.read();
-      while (b >= 0) {
-        hist[b] += 1l;
-        b = v.read();
-      }
-      v.close();
-      Mutation m = new Mutation(k.get(0).getKey().getRow());
-      m.put("info", "hist", cv, new Value(SummingArrayCombiner.STRING_ARRAY_ENCODER.encode(Arrays.asList(hist))));
-      context.write(new Text(), m);
-    }
-
-    @Override
-    protected void setup(Context context) throws IOException, InterruptedException {
-      cv = new ColumnVisibility(context.getConfiguration().get(VIS, ""));
-    }
-  }
-
-  static class Opts extends MapReduceClientOnRequiredTable {
-    @Parameter(names = "--vis")
-    String visibilities = "";
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    Job job = Job.getInstance(getConf());
-    job.setJobName(this.getClass().getSimpleName());
-    job.setJarByClass(this.getClass());
-
-    Opts opts = new Opts();
-    opts.parseArgs(CharacterHistogram.class.getName(), args);
-
-    job.setInputFormatClass(ChunkInputFormat.class);
-    opts.setAccumuloConfigs(job);
-    job.getConfiguration().set(VIS, opts.visibilities.toString());
-
-    job.setMapperClass(HistMapper.class);
-    job.setMapOutputKeyClass(Text.class);
-    job.setMapOutputValueClass(Mutation.class);
-
-    job.setNumReduceTasks(0);
-
-    job.setOutputFormatClass(AccumuloOutputFormat.class);
-
-    job.waitForCompletion(true);
-    return job.isSuccessful() ? 0 : 1;
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java
deleted file mode 100644
index 0ffeca0..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkCombiner.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.hadoop.io.Text;
-
-/**
- * This iterator dedupes chunks and sets their visibilities to the combined visibility of the refs columns. For example, it would combine
- *
- * <pre>
- *    row1 refs uid1\0a A&amp;B V0
- *    row1 refs uid2\0b C&amp;D V0
- *    row1 ~chunk 0 A&amp;B V1
- *    row1 ~chunk 0 C&amp;D V1
- *    row1 ~chunk 0 E&amp;F V1
- *    row1 ~chunk 0 G&amp;H V1
- * </pre>
- *
- * into the following
- *
- * <pre>
- *    row1 refs uid1\0a A&amp;B V0
- *    row1 refs uid2\0b C&amp;D V0
- *    row1 ~chunk 0 (A&amp;B)|(C&amp;D) V1
- * </pre>
- *
- * {@link VisibilityCombiner} is used to combie the visibilities.
- */
-
-public class ChunkCombiner implements SortedKeyValueIterator<Key,Value> {
-
-  private SortedKeyValueIterator<Key,Value> source;
-  private SortedKeyValueIterator<Key,Value> refsSource;
-  private static final Collection<ByteSequence> refsColf = Collections.singleton(FileDataIngest.REFS_CF_BS);
-  private Map<Text,byte[]> lastRowVC = Collections.emptyMap();
-
-  private Key topKey = null;
-  private Value topValue = null;
-
-  public ChunkCombiner() {}
-
-  @Override
-  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
-    this.source = source;
-    this.refsSource = source.deepCopy(env);
-  }
-
-  @Override
-  public boolean hasTop() {
-    return topKey != null;
-  }
-
-  @Override
-  public void next() throws IOException {
-    findTop();
-  }
-
-  @Override
-  public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
-    source.seek(range, columnFamilies, inclusive);
-    findTop();
-  }
-
-  private void findTop() throws IOException {
-    do {
-      topKey = null;
-      topValue = null;
-    } while (source.hasTop() && _findTop() == null);
-  }
-
-  private byte[] _findTop() throws IOException {
-    long maxTS;
-
-    topKey = new Key(source.getTopKey());
-    topValue = new Value(source.getTopValue());
-    source.next();
-
-    if (!topKey.getColumnFamilyData().equals(FileDataIngest.CHUNK_CF_BS))
-      return topKey.getColumnVisibility().getBytes();
-
-    maxTS = topKey.getTimestamp();
-
-    while (source.hasTop() && source.getTopKey().equals(topKey, PartialKey.ROW_COLFAM_COLQUAL)) {
-      if (source.getTopKey().getTimestamp() > maxTS)
-        maxTS = source.getTopKey().getTimestamp();
-
-      if (!topValue.equals(source.getTopValue()))
-        throw new RuntimeException("values not equals " + topKey + " " + source.getTopKey() + " : " + diffInfo(topValue, source.getTopValue()));
-
-      source.next();
-    }
-
-    byte[] vis = getVisFromRefs();
-    if (vis != null) {
-      topKey = new Key(topKey.getRowData().toArray(), topKey.getColumnFamilyData().toArray(), topKey.getColumnQualifierData().toArray(), vis, maxTS);
-    }
-    return vis;
-  }
-
-  private byte[] getVisFromRefs() throws IOException {
-    Text row = topKey.getRow();
-    if (lastRowVC.containsKey(row))
-      return lastRowVC.get(row);
-    Range range = new Range(row);
-    refsSource.seek(range, refsColf, true);
-    VisibilityCombiner vc = null;
-    while (refsSource.hasTop()) {
-      if (vc == null)
-        vc = new VisibilityCombiner();
-      vc.add(refsSource.getTopKey().getColumnVisibilityData());
-      refsSource.next();
-    }
-    if (vc == null) {
-      lastRowVC = Collections.singletonMap(row, null);
-      return null;
-    }
-    lastRowVC = Collections.singletonMap(row, vc.get());
-    return vc.get();
-  }
-
-  private String diffInfo(Value v1, Value v2) {
-    if (v1.getSize() != v2.getSize()) {
-      return "val len not equal " + v1.getSize() + "!=" + v2.getSize();
-    }
-
-    byte[] vb1 = v1.get();
-    byte[] vb2 = v2.get();
-
-    for (int i = 0; i < vb1.length; i++) {
-      if (vb1[i] != vb2[i]) {
-        return String.format("first diff at offset %,d 0x%02x != 0x%02x", i, 0xff & vb1[i], 0xff & vb2[i]);
-      }
-    }
-
-    return null;
-  }
-
-  @Override
-  public Key getTopKey() {
-    return topKey;
-  }
-
-  @Override
-  public Value getTopValue() {
-    return topValue;
-  }
-
-  @Override
-  public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
-    ChunkCombiner cc = new ChunkCombiner();
-    try {
-      cc.init(source.deepCopy(env), null, env);
-    } catch (IOException e) {
-      throw new IllegalArgumentException(e);
-    }
-    return cc;
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java
deleted file mode 100644
index bb7715b..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormat.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.mapreduce.InputFormatBase;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.PeekingIterator;
-import org.apache.accumulo.core.util.format.DefaultFormatter;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-/**
- * An InputFormat that turns the file data ingested with {@link FileDataIngest} into an InputStream using {@link ChunkInputStream}. Mappers used with this
- * InputFormat must close the InputStream.
- */
-public class ChunkInputFormat extends InputFormatBase<List<Entry<Key,Value>>,InputStream> {
-  @Override
-  public RecordReader<List<Entry<Key,Value>>,InputStream> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException,
-      InterruptedException {
-    return new RecordReaderBase<List<Entry<Key,Value>>,InputStream>() {
-      private PeekingIterator<Entry<Key,Value>> peekingScannerIterator;
-
-      @Override
-      public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
-        super.initialize(inSplit, attempt);
-        peekingScannerIterator = new PeekingIterator<>(scannerIterator);
-        currentK = new ArrayList<>();
-        currentV = new ChunkInputStream();
-      }
-
-      @Override
-      public boolean nextKeyValue() throws IOException, InterruptedException {
-        currentK.clear();
-        if (peekingScannerIterator.hasNext()) {
-          ++numKeysRead;
-          Entry<Key,Value> entry = peekingScannerIterator.peek();
-          while (!entry.getKey().getColumnFamily().equals(FileDataIngest.CHUNK_CF)) {
-            currentK.add(entry);
-            peekingScannerIterator.next();
-            if (!peekingScannerIterator.hasNext())
-              return true;
-            entry = peekingScannerIterator.peek();
-          }
-          currentKey = entry.getKey();
-          ((ChunkInputStream) currentV).setSource(peekingScannerIterator);
-          if (log.isTraceEnabled())
-            log.trace("Processing key/value pair: " + DefaultFormatter.formatEntry(entry, true));
-          return true;
-        }
-        return false;
-      }
-    };
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java
deleted file mode 100644
index 1774227..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStream.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.PeekingIterator;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An input stream that reads file data stored in one or more Accumulo values. Used by {@link ChunkInputFormat} to present input streams to a mapper.
- */
-public class ChunkInputStream extends InputStream {
-  private static final Logger log = LoggerFactory.getLogger(ChunkInputStream.class);
-
-  protected PeekingIterator<Entry<Key,Value>> source;
-  protected Key currentKey;
-  protected Set<Text> currentVis;
-  protected int currentChunk;
-  protected int currentChunkSize;
-  protected boolean gotEndMarker;
-
-  protected byte buf[];
-  protected int count;
-  protected int pos;
-
-  public ChunkInputStream() {
-    source = null;
-  }
-
-  public ChunkInputStream(PeekingIterator<Entry<Key,Value>> in) throws IOException {
-    setSource(in);
-  }
-
-  public void setSource(PeekingIterator<Entry<Key,Value>> in) throws IOException {
-    if (source != null)
-      throw new IOException("setting new source without closing old one");
-    this.source = in;
-    currentVis = new TreeSet<>();
-    count = pos = 0;
-    if (!source.hasNext()) {
-      log.debug("source has no next");
-      gotEndMarker = true;
-      return;
-    }
-
-    // read forward until we reach a chunk
-    Entry<Key,Value> entry = source.next();
-    currentKey = entry.getKey();
-    buf = entry.getValue().get();
-    while (!currentKey.getColumnFamily().equals(FileDataIngest.CHUNK_CF)) {
-      log.debug("skipping key: " + currentKey.toString());
-      if (!source.hasNext())
-        return;
-      entry = source.next();
-      currentKey = entry.getKey();
-      buf = entry.getValue().get();
-    }
-    log.debug("starting chunk: " + currentKey.toString());
-    count = buf.length;
-    currentVis.add(currentKey.getColumnVisibility());
-    currentChunk = FileDataIngest.bytesToInt(currentKey.getColumnQualifier().getBytes(), 4);
-    currentChunkSize = FileDataIngest.bytesToInt(currentKey.getColumnQualifier().getBytes(), 0);
-    gotEndMarker = false;
-    if (buf.length == 0)
-      gotEndMarker = true;
-    if (currentChunk != 0) {
-      source = null;
-      throw new IOException("starting chunk number isn't 0 for " + currentKey.getRow());
-    }
-  }
-
-  private int fill() throws IOException {
-    if (source == null || !source.hasNext()) {
-      if (gotEndMarker)
-        return count = pos = 0;
-      else
-        throw new IOException("no end chunk marker but source has no data");
-    }
-
-    Entry<Key,Value> entry = source.peek();
-    Key thisKey = entry.getKey();
-    log.debug("evaluating key: " + thisKey.toString());
-
-    // check that we're still on the same row
-    if (!thisKey.equals(currentKey, PartialKey.ROW)) {
-      if (gotEndMarker)
-        return -1;
-      else {
-        String currentRow = currentKey.getRow().toString();
-        clear();
-        throw new IOException("got to the end of the row without end chunk marker " + currentRow);
-      }
-    }
-    log.debug("matches current key");
-
-    // ok to advance the iterator
-    source.next();
-
-    // check that this is part of a chunk
-    if (!thisKey.getColumnFamily().equals(FileDataIngest.CHUNK_CF)) {
-      log.debug("skipping non-chunk key");
-      return fill();
-    }
-    log.debug("is a chunk");
-
-    // check that the chunk size is the same as the one being read
-    if (currentChunkSize != FileDataIngest.bytesToInt(thisKey.getColumnQualifier().getBytes(), 0)) {
-      log.debug("skipping chunk of different size");
-      return fill();
-    }
-
-    // add the visibility to the list if it's not there
-    if (!currentVis.contains(thisKey.getColumnVisibility()))
-      currentVis.add(thisKey.getColumnVisibility());
-
-    // check to see if it is an identical chunk with a different visibility
-    if (thisKey.getColumnQualifier().equals(currentKey.getColumnQualifier())) {
-      log.debug("skipping identical chunk with different visibility");
-      return fill();
-    }
-
-    if (gotEndMarker) {
-      log.debug("got another chunk after end marker: " + currentKey.toString() + " " + thisKey.toString());
-      clear();
-      throw new IOException("found extra chunk after end marker");
-    }
-
-    // got new chunk of the same file, check that it's the next chunk
-    int thisChunk = FileDataIngest.bytesToInt(thisKey.getColumnQualifier().getBytes(), 4);
-    if (thisChunk != currentChunk + 1) {
-      log.debug("new chunk same file, unexpected chunkID: " + currentKey.toString() + " " + thisKey.toString());
-      clear();
-      throw new IOException("missing chunks between " + currentChunk + " and " + thisChunk);
-    }
-
-    currentKey = thisKey;
-    currentChunk = thisChunk;
-    buf = entry.getValue().get();
-    pos = 0;
-
-    // check to see if it's the last chunk
-    if (buf.length == 0) {
-      gotEndMarker = true;
-      return fill();
-    }
-
-    return count = buf.length;
-  }
-
-  public Set<Text> getVisibilities() {
-    if (source != null)
-      throw new IllegalStateException("don't get visibilities before chunks have been completely read");
-    return currentVis;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (source == null)
-      return -1;
-    log.debug("pos: " + pos + " count: " + count);
-    if (pos >= count) {
-      if (fill() <= 0) {
-        log.debug("done reading input stream at key: " + (currentKey == null ? "null" : currentKey.toString()));
-        if (source != null && source.hasNext())
-          log.debug("next key: " + source.peek().getKey());
-        clear();
-        return -1;
-      }
-    }
-    return buf[pos++] & 0xff;
-  }
-
-  @Override
-  public int read(byte[] b, int off, int len) throws IOException {
-    if (b == null) {
-      throw new NullPointerException();
-    } else if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) {
-      throw new IndexOutOfBoundsException();
-    } else if (len == 0) {
-      return 0;
-    }
-
-    log.debug("filling buffer " + off + " " + len);
-    int total = 0;
-    while (total < len) {
-      int avail = count - pos;
-      log.debug(avail + " available in current local buffer");
-      if (avail <= 0) {
-        if (fill() <= 0) {
-          log.debug("done reading input stream at key: " + (currentKey == null ? "null" : currentKey.toString()));
-          if (source != null && source.hasNext())
-            log.debug("next key: " + source.peek().getKey());
-          clear();
-          log.debug("filled " + total + " bytes");
-          return total == 0 ? -1 : total;
-        }
-        avail = count - pos;
-      }
-
-      int cnt = (avail < len - total) ? avail : len - total;
-      log.debug("copying from local buffer: local pos " + pos + " into pos " + off + " len " + cnt);
-      System.arraycopy(buf, pos, b, off, cnt);
-      pos += cnt;
-      off += cnt;
-      total += cnt;
-    }
-    log.debug("filled " + total + " bytes");
-    return total;
-  }
-
-  public void clear() {
-    source = null;
-    buf = null;
-    currentKey = null;
-    currentChunk = 0;
-    pos = count = 0;
-  }
-
-  @Override
-  public void close() throws IOException {
-    try {
-      while (fill() > 0) {}
-    } catch (IOException e) {
-      clear();
-      throw new IOException(e);
-    }
-    clear();
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
deleted file mode 100644
index 1a0ec5d..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.data.ArrayByteSequence;
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Takes a list of files and archives them into Accumulo keyed on hashes of the files. See docs/examples/README.filedata for instructions.
- */
-public class FileDataIngest {
-  public static final Text CHUNK_CF = new Text("~chunk");
-  public static final Text REFS_CF = new Text("refs");
-  public static final String REFS_ORIG_FILE = "name";
-  public static final String REFS_FILE_EXT = "filext";
-  public static final ByteSequence CHUNK_CF_BS = new ArrayByteSequence(CHUNK_CF.getBytes(), 0, CHUNK_CF.getLength());
-  public static final ByteSequence REFS_CF_BS = new ArrayByteSequence(REFS_CF.getBytes(), 0, REFS_CF.getLength());
-
-  int chunkSize;
-  byte[] chunkSizeBytes;
-  byte[] buf;
-  MessageDigest md5digest;
-  ColumnVisibility cv;
-
-  public FileDataIngest(int chunkSize, ColumnVisibility colvis) {
-    this.chunkSize = chunkSize;
-    chunkSizeBytes = intToBytes(chunkSize);
-    buf = new byte[chunkSize];
-    try {
-      md5digest = MessageDigest.getInstance("MD5");
-    } catch (NoSuchAlgorithmException e) {
-      throw new RuntimeException(e);
-    }
-    cv = colvis;
-  }
-
-  public String insertFileData(String filename, BatchWriter bw) throws MutationsRejectedException, IOException {
-    if (chunkSize == 0)
-      return "";
-    md5digest.reset();
-    String uid = hexString(md5digest.digest(filename.getBytes()));
-
-    // read through file once, calculating hashes
-    md5digest.reset();
-    InputStream fis = null;
-    int numRead = 0;
-    try {
-      fis = new FileInputStream(filename);
-      numRead = fis.read(buf);
-      while (numRead >= 0) {
-        if (numRead > 0) {
-          md5digest.update(buf, 0, numRead);
-        }
-        numRead = fis.read(buf);
-      }
-    } finally {
-      if (fis != null) {
-        fis.close();
-      }
-    }
-
-    String hash = hexString(md5digest.digest());
-    Text row = new Text(hash);
-
-    // write info to accumulo
-    Mutation m = new Mutation(row);
-    m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_ORIG_FILE), cv, new Value(filename.getBytes()));
-    String fext = getExt(filename);
-    if (fext != null)
-      m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_FILE_EXT), cv, new Value(fext.getBytes()));
-    bw.addMutation(m);
-
-    // read through file again, writing chunks to accumulo
-    int chunkCount = 0;
-    try {
-      fis = new FileInputStream(filename);
-      numRead = fis.read(buf);
-      while (numRead >= 0) {
-        while (numRead < buf.length) {
-          int moreRead = fis.read(buf, numRead, buf.length - numRead);
-          if (moreRead > 0)
-            numRead += moreRead;
-          else if (moreRead < 0)
-            break;
-        }
-        m = new Mutation(row);
-        Text chunkCQ = new Text(chunkSizeBytes);
-        chunkCQ.append(intToBytes(chunkCount), 0, 4);
-        m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead));
-        bw.addMutation(m);
-        if (chunkCount == Integer.MAX_VALUE)
-          throw new RuntimeException("too many chunks for file " + filename + ", try raising chunk size");
-        chunkCount++;
-        numRead = fis.read(buf);
-      }
-    } finally {
-      if (fis != null) {
-        fis.close();
-      }
-    }
-    m = new Mutation(row);
-    Text chunkCQ = new Text(chunkSizeBytes);
-    chunkCQ.append(intToBytes(chunkCount), 0, 4);
-    m.put(new Text(CHUNK_CF), chunkCQ, cv, new Value(new byte[0]));
-    bw.addMutation(m);
-    return hash;
-  }
-
-  public static int bytesToInt(byte[] b, int offset) {
-    if (b.length <= offset + 3)
-      throw new NumberFormatException("couldn't pull integer from bytes at offset " + offset);
-    int i = (((b[offset] & 255) << 24) + ((b[offset + 1] & 255) << 16) + ((b[offset + 2] & 255) << 8) + ((b[offset + 3] & 255) << 0));
-    return i;
-  }
-
-  public static byte[] intToBytes(int l) {
-    byte[] b = new byte[4];
-    b[0] = (byte) (l >>> 24);
-    b[1] = (byte) (l >>> 16);
-    b[2] = (byte) (l >>> 8);
-    b[3] = (byte) (l >>> 0);
-    return b;
-  }
-
-  private static String getExt(String filename) {
-    if (filename.indexOf(".") == -1)
-      return null;
-    return filename.substring(filename.lastIndexOf(".") + 1);
-  }
-
-  public String hexString(byte[] bytes) {
-    StringBuilder sb = new StringBuilder();
-    for (byte b : bytes) {
-      sb.append(String.format("%02x", b));
-    }
-    return sb.toString();
-  }
-
-  public static class Opts extends ClientOnRequiredTable {
-    @Parameter(names = "--vis", description = "use a given visibility for the new counts", converter = VisibilityConverter.class)
-    ColumnVisibility visibility = new ColumnVisibility();
-
-    @Parameter(names = "--chunk", description = "size of the chunks used to store partial files")
-    int chunkSize = 64 * 1024;
-
-    @Parameter(description = "<file> { <file> ... }")
-    List<String> files = new ArrayList<>();
-  }
-
-  public static void main(String[] args) throws Exception {
-    Opts opts = new Opts();
-    BatchWriterOpts bwOpts = new BatchWriterOpts();
-    opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);
-
-    Connector conn = opts.getConnector();
-    if (!conn.tableOperations().exists(opts.getTableName())) {
-      conn.tableOperations().create(opts.getTableName());
-      conn.tableOperations().attachIterator(opts.getTableName(), new IteratorSetting(1, ChunkCombiner.class));
-    }
-    BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
-    FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
-    for (String filename : opts.files) {
-      fdi.insertFileData(filename, bw);
-    }
-    bw.close();
-    opts.stopTracing();
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
deleted file mode 100644
index 48746d0..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.PeekingIterator;
-
-/**
- * Retrieves file data based on the hash of the file. Used by the {@link org.apache.accumulo.examples.simple.dirlist.Viewer}. See README.dirlist for
- * instructions.
- */
-public class FileDataQuery {
-  private Connector conn = null;
-  List<Entry<Key,Value>> lastRefs;
-  private ChunkInputStream cis;
-  Scanner scanner;
-
-  public FileDataQuery(String instanceName, String zooKeepers, String user, AuthenticationToken token, String tableName, Authorizations auths)
-      throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
-    ZooKeeperInstance instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zooKeepers));
-    conn = instance.getConnector(user, token);
-    lastRefs = new ArrayList<>();
-    cis = new ChunkInputStream();
-    scanner = conn.createScanner(tableName, auths);
-  }
-
-  public List<Entry<Key,Value>> getLastRefs() {
-    return lastRefs;
-  }
-
-  public ChunkInputStream getData(String hash) throws IOException {
-    scanner.setRange(new Range(hash));
-    scanner.setBatchSize(1);
-    lastRefs.clear();
-    PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(scanner.iterator());
-    if (pi.hasNext()) {
-      while (!pi.peek().getKey().getColumnFamily().equals(FileDataIngest.CHUNK_CF)) {
-        lastRefs.add(pi.peek());
-        pi.next();
-      }
-    }
-    cis.clear();
-    cis.setSource(pi);
-    return cis;
-  }
-
-  public String getSomeData(String hash, int numBytes) throws IOException {
-    ChunkInputStream is = getData(hash);
-    byte[] buf = new byte[numBytes];
-    if (is.read(buf) >= 0) {
-      return new String(buf);
-    } else {
-      return "";
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/KeyUtil.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/KeyUtil.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/KeyUtil.java
deleted file mode 100644
index f9c52ba..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/KeyUtil.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.util.ArrayList;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * A utility for creating and parsing null-byte separated strings into/from Text objects.
- */
-public class KeyUtil {
-  public static final byte[] nullbyte = new byte[] {0};
-
-  /**
-   * Join some number of strings using a null byte separator into a text object.
-   *
-   * @param s
-   *          strings
-   * @return a text object containing the strings separated by null bytes
-   */
-  public static Text buildNullSepText(String... s) {
-    Text t = new Text(s[0]);
-    for (int i = 1; i < s.length; i++) {
-      t.append(nullbyte, 0, 1);
-      t.append(s[i].getBytes(), 0, s[i].length());
-    }
-    return t;
-  }
-
-  /**
-   * Split a text object using a null byte separator into an array of strings.
-   *
-   * @param t
-   *          null-byte separated text object
-   * @return an array of strings
-   */
-  public static String[] splitNullSepText(Text t) {
-    ArrayList<String> s = new ArrayList<>();
-    byte[] b = t.getBytes();
-    int lastindex = 0;
-    for (int i = 0; i < t.getLength(); i++) {
-      if (b[i] == (byte) 0) {
-        s.add(new String(b, lastindex, i - lastindex));
-        lastindex = i + 1;
-      }
-    }
-    s.add(new String(b, lastindex, t.getLength() - lastindex));
-    return s.toArray(new String[s.size()]);
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/VisibilityCombiner.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/VisibilityCombiner.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/VisibilityCombiner.java
deleted file mode 100644
index b205ec1..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/VisibilityCombiner.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.data.ByteSequence;
-
-/**
- * A utility for merging visibilities into the form {@code (VIS1)|(VIS2)|...|(VISN)}. Used by the {@link ChunkCombiner}.
- */
-public class VisibilityCombiner {
-
-  private TreeSet<String> visibilities = new TreeSet<>();
-
-  void add(ByteSequence cv) {
-    if (cv.length() == 0)
-      return;
-
-    int depth = 0;
-    int offset = 0;
-
-    for (int i = 0; i < cv.length(); i++) {
-      switch (cv.byteAt(i)) {
-        case '(':
-          depth++;
-          break;
-        case ')':
-          depth--;
-          if (depth < 0)
-            throw new IllegalArgumentException("Invalid vis " + cv);
-          break;
-        case '|':
-          if (depth == 0) {
-            insert(cv.subSequence(offset, i));
-            offset = i + 1;
-          }
-
-          break;
-      }
-    }
-
-    insert(cv.subSequence(offset, cv.length()));
-
-    if (depth != 0)
-      throw new IllegalArgumentException("Invalid vis " + cv);
-
-  }
-
-  private void insert(ByteSequence cv) {
-    for (int i = 0; i < cv.length(); i++) {
-
-    }
-
-    String cvs = cv.toString();
-
-    if (cvs.charAt(0) != '(')
-      cvs = "(" + cvs + ")";
-    else {
-      int depth = 0;
-      int depthZeroCloses = 0;
-      for (int i = 0; i < cv.length(); i++) {
-        switch (cv.byteAt(i)) {
-          case '(':
-            depth++;
-            break;
-          case ')':
-            depth--;
-            if (depth == 0)
-              depthZeroCloses++;
-            break;
-        }
-      }
-
-      if (depthZeroCloses > 1)
-        cvs = "(" + cvs + ")";
-    }
-
-    visibilities.add(cvs);
-  }
-
-  byte[] get() {
-    StringBuilder sb = new StringBuilder();
-    String sep = "";
-    for (String cvs : visibilities) {
-      sb.append(sep);
-      sep = "|";
-      sb.append(cvs);
-    }
-
-    return sb.toString().getBytes();
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java
deleted file mode 100644
index 0e60086..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithBatchWriter.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.helloworld;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MultiTableBatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-/**
- * Inserts 10K rows (50K entries) into accumulo with each row having 5 entries.
- */
-public class InsertWithBatchWriter {
-
-  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, MutationsRejectedException, TableExistsException,
-      TableNotFoundException {
-    ClientOnRequiredTable opts = new ClientOnRequiredTable();
-    BatchWriterOpts bwOpts = new BatchWriterOpts();
-    opts.parseArgs(InsertWithBatchWriter.class.getName(), args, bwOpts);
-
-    Connector connector = opts.getConnector();
-    MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
-
-    if (!connector.tableOperations().exists(opts.getTableName()))
-      connector.tableOperations().create(opts.getTableName());
-    BatchWriter bw = mtbw.getBatchWriter(opts.getTableName());
-
-    Text colf = new Text("colfam");
-    System.out.println("writing ...");
-    for (int i = 0; i < 10000; i++) {
-      Mutation m = new Mutation(new Text(String.format("row_%d", i)));
-      for (int j = 0; j < 5; j++) {
-        m.put(colf, new Text(String.format("colqual_%d", j)), new Value((String.format("value_%d_%d", i, j)).getBytes()));
-      }
-      bw.addMutation(m);
-      if (i % 100 == 0)
-        System.out.println(i);
-    }
-    mtbw.close();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java
deleted file mode 100644
index c8c984d..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/ReadData.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.helloworld;
-
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Reads all data between two rows; all data after a given row; or all data in a table, depending on the number of arguments given.
- */
-public class ReadData {
-
-  private static final Logger log = LoggerFactory.getLogger(ReadData.class);
-
-  static class Opts extends ClientOnRequiredTable {
-    @Parameter(names = "--startKey")
-    String startKey;
-    @Parameter(names = "--endKey")
-    String endKey;
-  }
-
-  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
-    Opts opts = new Opts();
-    ScannerOpts scanOpts = new ScannerOpts();
-    opts.parseArgs(ReadData.class.getName(), args, scanOpts);
-
-    Connector connector = opts.getConnector();
-
-    Scanner scan = connector.createScanner(opts.getTableName(), opts.auths);
-    scan.setBatchSize(scanOpts.scanBatchSize);
-    Key start = null;
-    if (opts.startKey != null)
-      start = new Key(new Text(opts.startKey));
-    Key end = null;
-    if (opts.endKey != null)
-      end = new Key(new Text(opts.endKey));
-    scan.setRange(new Range(start, end));
-    Iterator<Entry<Key,Value>> iter = scan.iterator();
-
-    while (iter.hasNext()) {
-      Entry<Key,Value> e = iter.next();
-      Text colf = e.getKey().getColumnFamily();
-      Text colq = e.getKey().getColumnQualifier();
-      log.trace("row: " + e.getKey().getRow() + ", colf: " + colf + ", colq: " + colq);
-      log.trace(", value: " + e.getValue().toString());
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
deleted file mode 100644
index a2afcdf..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.isolation;
-
-import java.util.HashSet;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IsolatedScanner;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * This example shows how a concurrent reader and writer can interfere with each other. It creates two threads that run forever reading and writing to the same
- * table.
- *
- * When the example is run with isolation enabled, no interference will be observed.
- *
- * When the example is run with out isolation, the reader will see partial mutations of a row.
- *
- */
-
-public class InterferenceTest {
-
-  private static final int NUM_ROWS = 500;
-  private static final int NUM_COLUMNS = 113; // scanner batches 1000 by default, so make num columns not a multiple of 10
-  private static final Logger log = LoggerFactory.getLogger(InterferenceTest.class);
-
-  static class Writer implements Runnable {
-
-    private final BatchWriter bw;
-    private final long iterations;
-
-    Writer(BatchWriter bw, long iterations) {
-      this.bw = bw;
-      this.iterations = iterations;
-    }
-
-    @Override
-    public void run() {
-      int row = 0;
-      int value = 0;
-
-      for (long i = 0; i < iterations; i++) {
-        Mutation m = new Mutation(new Text(String.format("%03d", row)));
-        row = (row + 1) % NUM_ROWS;
-
-        for (int cq = 0; cq < NUM_COLUMNS; cq++)
-          m.put(new Text("000"), new Text(String.format("%04d", cq)), new Value(("" + value).getBytes()));
-
-        value++;
-
-        try {
-          bw.addMutation(m);
-        } catch (MutationsRejectedException e) {
-          log.error("Mutation was rejected.", e);
-          System.exit(-1);
-        }
-      }
-      try {
-        bw.close();
-      } catch (MutationsRejectedException e) {
-        log.error("Mutation was rejected on BatchWriter close.", e);
-      }
-    }
-  }
-
-  static class Reader implements Runnable {
-
-    private Scanner scanner;
-    volatile boolean stop = false;
-
-    Reader(Scanner scanner) {
-      this.scanner = scanner;
-    }
-
-    @Override
-    public void run() {
-      while (!stop) {
-        ByteSequence row = null;
-        int count = 0;
-
-        // all columns in a row should have the same value,
-        // use this hash set to track that
-        HashSet<String> values = new HashSet<>();
-
-        for (Entry<Key,Value> entry : scanner) {
-          if (row == null)
-            row = entry.getKey().getRowData();
-
-          if (!row.equals(entry.getKey().getRowData())) {
-            if (count != NUM_COLUMNS)
-              System.err.println("ERROR Did not see " + NUM_COLUMNS + " columns in row " + row);
-
-            if (values.size() > 1)
-              System.err.println("ERROR Columns in row " + row + " had multiple values " + values);
-
-            row = entry.getKey().getRowData();
-            count = 0;
-            values.clear();
-          }
-
-          count++;
-
-          values.add(entry.getValue().toString());
-        }
-
-        if (count > 0 && count != NUM_COLUMNS)
-          System.err.println("ERROR Did not see " + NUM_COLUMNS + " columns in row " + row);
-
-        if (values.size() > 1)
-          System.err.println("ERROR Columns in row " + row + " had multiple values " + values);
-      }
-    }
-
-    public void stopNow() {
-      stop = true;
-    }
-  }
-
-  static class Opts extends ClientOnRequiredTable {
-    @Parameter(names = "--iterations", description = "number of times to run", required = true)
-    long iterations = 0;
-    @Parameter(names = "--isolated", description = "use isolated scans")
-    boolean isolated = false;
-  }
-
-  public static void main(String[] args) throws Exception {
-    Opts opts = new Opts();
-    BatchWriterOpts bwOpts = new BatchWriterOpts();
-    opts.parseArgs(InterferenceTest.class.getName(), args, bwOpts);
-
-    if (opts.iterations < 1)
-      opts.iterations = Long.MAX_VALUE;
-
-    Connector conn = opts.getConnector();
-    if (!conn.tableOperations().exists(opts.getTableName()))
-      conn.tableOperations().create(opts.getTableName());
-
-    Thread writer = new Thread(new Writer(conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig()), opts.iterations));
-    writer.start();
-    Reader r;
-    if (opts.isolated)
-      r = new Reader(new IsolatedScanner(conn.createScanner(opts.getTableName(), opts.auths)));
-    else
-      r = new Reader(conn.createScanner(opts.getTableName(), opts.auths));
-    Thread reader;
-    reader = new Thread(r);
-    reader.start();
-    writer.join();
-    r.stopNow();
-    reader.join();
-    System.out.println("finished");
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
deleted file mode 100644
index 3355454..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.mapreduce;
-
-import java.io.IOException;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.cli.MapReduceClientOnRequiredTable;
-import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Map job to ingest n-gram files from http://storage.googleapis.com/books/ngrams/books/datasetsv2.html
- */
-public class NGramIngest extends Configured implements Tool {
-
-  private static final Logger log = LoggerFactory.getLogger(NGramIngest.class);
-
-  static class Opts extends MapReduceClientOnRequiredTable {
-    @Parameter(names = "--input", required = true)
-    String inputDirectory;
-  }
-
-  static class NGramMapper extends Mapper<LongWritable,Text,Text,Mutation> {
-
-    @Override
-    protected void map(LongWritable location, Text value, Context context) throws IOException, InterruptedException {
-      String parts[] = value.toString().split("\\t");
-      if (parts.length >= 4) {
-        Mutation m = new Mutation(parts[0]);
-        m.put(parts[1], String.format("%010d", Long.parseLong(parts[2])), new Value(parts[3].trim().getBytes()));
-        context.write(null, m);
-      }
-    }
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    Opts opts = new Opts();
-    opts.parseArgs(getClass().getName(), args);
-
-    Job job = Job.getInstance(getConf());
-    job.setJobName(getClass().getSimpleName());
-    job.setJarByClass(getClass());
-
-    opts.setAccumuloConfigs(job);
-    job.setInputFormatClass(TextInputFormat.class);
-    job.setOutputFormatClass(AccumuloOutputFormat.class);
-
-    job.setMapperClass(NGramMapper.class);
-    job.setMapOutputKeyClass(Text.class);
-    job.setMapOutputValueClass(Mutation.class);
-
-    job.setNumReduceTasks(0);
-    job.setSpeculativeExecution(false);
-
-    if (!opts.getConnector().tableOperations().exists(opts.getTableName())) {
-      log.info("Creating table " + opts.getTableName());
-      opts.getConnector().tableOperations().create(opts.getTableName());
-      SortedSet<Text> splits = new TreeSet<>();
-      String numbers[] = "1 2 3 4 5 6 7 8 9".split("\\s");
-      String lower[] = "a b c d e f g h i j k l m n o p q r s t u v w x y z".split("\\s");
-      String upper[] = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split("\\s");
-      for (String[] array : new String[][] {numbers, lower, upper}) {
-        for (String s : array) {
-          splits.add(new Text(s));
-        }
-      }
-      opts.getConnector().tableOperations().addSplits(opts.getTableName(), splits);
-    }
-
-    TextInputFormat.addInputPath(job, new Path(opts.inputDirectory));
-    job.waitForCompletion(true);
-    return job.isSuccessful() ? 0 : 1;
-  }
-
-  public static void main(String[] args) throws Exception {
-    int res = ToolRunner.run(new Configuration(), new NGramIngest(), args);
-    if (res != 0)
-      System.exit(res);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java
deleted file mode 100644
index ad5bc74..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.mapreduce;
-
-import java.io.IOException;
-
-import org.apache.accumulo.core.cli.MapReduceClientOnRequiredTable;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.RegExFilter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import com.beust.jcommander.Parameter;
-
-public class RegexExample extends Configured implements Tool {
-  public static class RegexMapper extends Mapper<Key,Value,Key,Value> {
-    @Override
-    public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
-      context.write(row, data);
-    }
-  }
-
-  static class Opts extends MapReduceClientOnRequiredTable {
-    @Parameter(names = "--rowRegex")
-    String rowRegex;
-    @Parameter(names = "--columnFamilyRegex")
-    String columnFamilyRegex;
-    @Parameter(names = "--columnQualifierRegex")
-    String columnQualifierRegex;
-    @Parameter(names = "--valueRegex")
-    String valueRegex;
-    @Parameter(names = "--output", required = true)
-    String destination;
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    Opts opts = new Opts();
-    opts.parseArgs(getClass().getName(), args);
-
-    Job job = Job.getInstance(getConf());
-    job.setJobName(getClass().getSimpleName());
-    job.setJarByClass(getClass());
-
-    job.setInputFormatClass(AccumuloInputFormat.class);
-    opts.setAccumuloConfigs(job);
-
-    IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
-    RegExFilter.setRegexs(regex, opts.rowRegex, opts.columnFamilyRegex, opts.columnQualifierRegex, opts.valueRegex, false);
-    AccumuloInputFormat.addIterator(job, regex);
-
-    job.setMapperClass(RegexMapper.class);
-    job.setMapOutputKeyClass(Key.class);
-    job.setMapOutputValueClass(Value.class);
-
-    job.setNumReduceTasks(0);
-
-    job.setOutputFormatClass(TextOutputFormat.class);
-    TextOutputFormat.setOutputPath(job, new Path(opts.destination));
-
-    System.out.println("setRowRegex: " + opts.rowRegex);
-    System.out.println("setColumnFamilyRegex: " + opts.columnFamilyRegex);
-    System.out.println("setColumnQualifierRegex: " + opts.columnQualifierRegex);
-    System.out.println("setValueRegex: " + opts.valueRegex);
-
-    job.waitForCompletion(true);
-    return job.isSuccessful() ? 0 : 1;
-  }
-
-  public static void main(String[] args) throws Exception {
-    int res = ToolRunner.run(new Configuration(), new RegexExample(), args);
-    if (res != 0)
-      System.exit(res);
-  }
-}


Mime
View raw message