accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch master updated: ACCUMULO-4772 Update shell to use NewTableConfiguration methods (#370)
Date Wed, 07 Feb 2018 14:59:54 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 4c0d7de  ACCUMULO-4772 Update shell to use NewTableConfiguration methods (#370)
4c0d7de is described below

commit 4c0d7dedf25eb1bb34bc5e1184978bbda00e86c3
Author: Mark Owens <jmarkowe@gmail.com>
AuthorDate: Wed Feb 7 09:59:51 2018 -0500

    ACCUMULO-4772 Update shell to use NewTableConfiguration methods (#370)
    
    Update Accumulo shell to utilize new NewTableConfiguration methods.
    
    Added capability to pre-configure iterators and locality groups when creating
    new tables in Accumulo. Updated CreateTableCommand.java to accept locality group
    and iterator data. Updated ShellServerIT.java to test the new capabilities.
---
 .../shell/commands/CreateTableCommand.java         | 126 +++++++++++++++-
 .../org/apache/accumulo/test/ShellServerIT.java    | 158 +++++++++++++++++++++
 2 files changed, 281 insertions(+), 3 deletions(-)

diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
index eac16fa..5a66de0 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
@@ -17,7 +17,11 @@
 package org.apache.accumulo.shell.commands;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -26,6 +30,7 @@ import java.util.TreeSet;
 
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.admin.NewTableConfiguration;
@@ -34,6 +39,7 @@ import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.constraints.VisibilityConstraint;
 import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.shell.Shell;
 import org.apache.accumulo.shell.Shell.Command;
 import org.apache.accumulo.shell.ShellUtil;
@@ -56,6 +62,8 @@ public class CreateTableCommand extends Command {
   private Option base64Opt;
   private Option createTableOptFormatter;
   private Option createTableOptInitProp;
+  private Option createTableOptLocalityProps;
+  private Option createTableOptIteratorProps;
 
   @Override
   public int execute(final String fullCommand, final CommandLine cl, final Shell shellState)
throws AccumuloException, AccumuloSecurityException,
@@ -63,6 +71,7 @@ public class CreateTableCommand extends Command {
 
     final String testTableName = cl.getArgs()[0];
     final HashMap<String,String> props = new HashMap<>();
+    NewTableConfiguration ntc = new NewTableConfiguration();
 
     if (!testTableName.matches(Tables.VALID_NAME_REGEX)) {
       shellState.getReader().println("Only letters, numbers and underscores are allowed for
use in table names.");
@@ -106,8 +115,18 @@ public class CreateTableCommand extends Command {
       }
     }
 
+    // Set iterator if supplied
+    if (cl.hasOption(createTableOptIteratorProps.getOpt())) {
+      ntc = attachIteratorToNewTable(cl, shellState, ntc);
+    }
+
+    // Set up locality groups, if supplied
+    if (cl.hasOption(createTableOptLocalityProps.getOpt())) {
+      ntc = setLocalityForNewTable(cl, ntc);
+    }
+
     // create table
-    shellState.getConnector().tableOperations().create(tableName, new NewTableConfiguration().setTimeType(timeType).setProperties(props));
+    shellState.getConnector().tableOperations().create(tableName, ntc.setTimeType(timeType).setProperties(props));
     if (partitions.size() > 0) {
       shellState.getConnector().tableOperations().addSplits(tableName, partitions);
     }
@@ -150,9 +169,100 @@ public class CreateTableCommand extends Command {
     return 0;
   }
 
+  /**
+   * Add supplied locality groups information to a NewTableConfiguration object.
+   *
+   * Used in conjunction with createtable shell command to allow locality groups to be configured
upon table creation.
+   */
+  private NewTableConfiguration setLocalityForNewTable(CommandLine cl, NewTableConfiguration
ntc) {
+    HashMap<String,Set<Text>> localityGroupMap = new HashMap<>();
+    String[] options = cl.getOptionValues(createTableOptLocalityProps.getOpt());
+    for (String localityInfo : options) {
+      final String parts[] = localityInfo.split("=", 2);
+      if (parts.length < 2)
+        throw new IllegalArgumentException("Missing '=' or there are spaces between entries");
+      final String groupName = parts[0];
+      final HashSet<Text> colFams = new HashSet<>();
+      for (String family : parts[1].split(","))
+        colFams.add(new Text(family.getBytes(Shell.CHARSET)));
+      // check that group names are not duplicated on usage line
+      if (localityGroupMap.put(groupName, colFams) != null)
+        throw new IllegalArgumentException("Duplicate locality group name found. Group names
must be unique");
+    }
+    ntc.setLocalityGroups(localityGroupMap);
+    return ntc;
+  }
+
+  /**
+   * Add supplied iterator information to NewTableConfiguration object.
+   *
+   * Used in conjunction with createtable shell command to allow an iterator to be configured
upon table creation.
+   */
+  private NewTableConfiguration attachIteratorToNewTable(final CommandLine cl, final Shell
shellState, NewTableConfiguration ntc) {
+    EnumSet<IteratorScope> scopeEnumSet;
+    IteratorSetting iteratorSetting;
+    if (shellState.iteratorProfiles.size() == 0)
+      throw new IllegalArgumentException("No shell iterator profiles have been created.");
+    String[] options = cl.getOptionValues(createTableOptIteratorProps.getOpt());
+    for (String profileInfo : options) {
+      String[] parts = profileInfo.split(":", 2);
+      String profileName = parts[0];
+      // The iteratorProfiles.get calls below will throw an NPE if the profile does not exist
+      // This can occur when the profile actually does not exist or if there is
+      // extraneous spacing in the iterator profile argument list causing the parser to read
a scope as a profile.
+      try {
+        iteratorSetting = shellState.iteratorProfiles.get(profileName).get(0);
+      } catch (NullPointerException ex) {
+        throw new IllegalArgumentException("invalid iterator argument. Either profile does
not exist or unexpected spaces in argument list.", ex);
+      }
+      // handle case where only the profile is supplied. Use all scopes by default if no
scope args are provided.
+      if (parts.length == 1) {
+        // add all scopes to enum set
+        scopeEnumSet = EnumSet.allOf(IteratorScope.class);
+      } else {
+        // user provided scope arguments exist, parse them
+        List<String> scopeArgs = Arrays.asList(parts[1].split(","));
+        // there are only three allowable scope values
+        if (scopeArgs.size() > 3)
+          throw new IllegalArgumentException("Too many scope arguments supplied");
+        // handle the 'all' argument separately since it is not an allowable enum value for
IteratorScope
+        // if 'all' is used, it should be the only scope provided
+        if (scopeArgs.contains("all")) {
+          if (scopeArgs.size() > 1)
+            throw new IllegalArgumentException("Cannot use 'all' in conjunction with other
scopes");
+          scopeEnumSet = EnumSet.allOf(IteratorScope.class);
+        } else {
+          // 'all' is not involved, examine the scope arguments and populate iterator scope
EnumSet
+          scopeEnumSet = validateScopes(scopeArgs);
+        }
+      }
+      ntc.attachIterator(iteratorSetting, scopeEnumSet);
+    }
+    return ntc;
+  }
+
+  /**
+   * Validate that the provided scope arguments are valid iterator scope settings. Checking
for duplicate entries and invalid scope values.
+   *
+   * Returns an EnumSet of scopes to be set.
+   */
+  private EnumSet<IteratorScope> validateScopes(final List<String> scopeList)
{
+    EnumSet<IteratorScope> scopes = EnumSet.noneOf(IteratorScope.class);
+    for (String scopeStr : scopeList) {
+      try {
+        IteratorScope scope = IteratorScope.valueOf(scopeStr);
+        if (!scopes.add(scope))
+          throw new IllegalArgumentException("duplicate scope arguments found");
+      } catch (IllegalArgumentException ex) {
+        throw new IllegalArgumentException("illegal scope arguments: " + ex.getMessage(),
ex);
+      }
+    }
+    return scopes;
+  }
+
   @Override
   public String description() {
-    return "creates a new table, with optional aggregators and optionally pre-split";
+    return "creates a new table, with optional aggregators, iterators, locality groups and
optionally pre-split";
   }
 
   @Override
@@ -174,13 +284,21 @@ public class CreateTableCommand extends Command {
         "prevent users from writing data they cannot read.  When enabling this, consider
disabling bulk import and alter table.");
     createTableOptFormatter = new Option("f", "formatter", true, "default formatter to set");
     createTableOptInitProp = new Option("prop", "init-properties", true, "user defined initial
properties");
-
     createTableOptCopyConfig.setArgName("table");
     createTableOptCopySplits.setArgName("table");
     createTableOptSplit.setArgName("filename");
     createTableOptFormatter.setArgName("className");
     createTableOptInitProp.setArgName("properties");
 
+    createTableOptLocalityProps = new Option("l", "locality", true, "create locality groups
at table creation");
+    createTableOptLocalityProps.setArgName("group=col_fam[,col_fam]");
+    createTableOptLocalityProps.setArgs(Option.UNLIMITED_VALUES);
+
+    createTableOptIteratorProps = new Option("i", "iter", true,
+        "initialize iterator at table creation using profile. If no scope supplied, all scopes
are activated.");
+    createTableOptIteratorProps.setArgName("profile[:[all]|[scan[,]][minc[,]][majc]]");
+    createTableOptIteratorProps.setArgs(Option.UNLIMITED_VALUES);
+
     // Splits and CopySplits are put in an optionsgroup to make them
     // mutually exclusive
     final OptionGroup splitOrCopySplit = new OptionGroup();
@@ -202,6 +320,8 @@ public class CreateTableCommand extends Command {
     o.addOption(createTableOptEVC);
     o.addOption(createTableOptFormatter);
     o.addOption(createTableOptInitProp);
+    o.addOption(createTableOptLocalityProps);
+    o.addOption(createTableOptIteratorProps);
 
     return o;
   }
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index 0a91cc1..3e10909 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -34,11 +34,13 @@ import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.Constants;
@@ -2020,4 +2022,160 @@ public class ShellServerIT extends SharedMiniClusterBase {
     // check that there are two files, with none having extra summary info
     assertMatches(output, "(?sm).*^.*total[:]2[,]\\s+missing[:]0[,]\\s+extra[:]0.*$.*");
   }
+
+  @Test
+  public void testCreateTableWithLocalityGroups() throws Exception {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2", true);
+    Connector connector = getConnector();
+    Map<String,Set<Text>> lMap = connector.tableOperations().getLocalityGroups(table);
+    Set<Text> expectedColFams = new HashSet<>(Arrays.asList(new Text("fam1"),
new Text("fam2")));
+    for (Entry<String,Set<Text>> entry : lMap.entrySet()) {
+      Assert.assertTrue(entry.getKey().equals("locg1"));
+      Assert.assertTrue(entry.getValue().containsAll(expectedColFams));
+    }
+    ts.exec("deletetable -f " + table);
+  }
+
+  /**
+   * Due to the existing complexity of the createtable command, the createtable help only
displays an example of setting one locality group. It is possible to
+   * set multiple groups if needed. This test verifies that capability.
+   */
+  @Test
+  public void testCreateTableWithMultipleLocalityGroups() throws Exception {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=colfam1", true);
+    Connector connector = getConnector();
+    Map<String,Set<Text>> lMap = connector.tableOperations().getLocalityGroups(table);
+    Assert.assertTrue(lMap.keySet().contains("locg1"));
+    Assert.assertTrue(lMap.keySet().contains("locg2"));
+    Set<Text> expectedColFams1 = new HashSet<>(Arrays.asList(new Text("fam1"),
new Text("fam2")));
+    Set<Text> expectedColFams2 = new HashSet<>(Arrays.asList(new Text("colfam1")));
+    Assert.assertTrue(lMap.get("locg1").containsAll(expectedColFams1));
+    Assert.assertTrue(lMap.get("locg2").containsAll(expectedColFams2));
+    ts.exec("deletetable -f " + table);
+  }
+
+  @Test
+  public void testCreateTableWithLocalityGroupsBadArguments() throws IOException {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table + " -l locg1 fam1,fam2", false);
+    ts.exec("createtable " + table + "-l", false);
+    ts.exec("createtable " + table + " -l locg1 = fam1,fam2", false);
+    ts.exec("createtable " + table + " -l locg1=fam1 ,fam2", false);
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg1=fam3,fam4", false);
+    ts.exec("createtable " + table + " -l locg1=fam1,fam2 locg2=fam1", false);
+    ts.exec("createtable " + table + " -l locg1", false);
+    ts.exec("createtable " + table + " group=fam1", false);
+    ts.exec("createtable " + table + "-l fam1,fam2", false);
+    ts.exec("createtable " + table + " -local locg1=fam1,fam2", false);
+  }
+
+  @Test
+  public void testCreateTableWithIterators() throws Exception {
+    final String tmpTable = "tmpTable";
+    final String table = name.getMethodName();
+
+    // create iterator profile
+    // Will use tmpTable for creating profile since setshelliter is requiring a table
+    // even though its command line help indicates that it is optional. Likely due to
+    // the fact that setshelliter extends setiter, which does require a table argument.
+    ts.exec("createtable " + tmpTable, true);
+    String output = ts.exec("tables");
+    Assert.assertTrue(output.contains(tmpTable));
+
+    ts.input.set("\n5000\n\n");
+    ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile1"));
+
+    // create table making use of the iterator profile
+    ts.exec("createtable " + table + " -i profile1:scan,minc", true);
+    ts.exec("insert foo a b c", true);
+    ts.exec("scan", true, "foo a:b []    c");
+    ts.exec("sleep 6", true);
+    ts.exec("scan", true, "", true);
+    ts.exec("deletetable -f " + table);
+    ts.exec("deletetable -f " + tmpTable);
+  }
+
+  /**
+   * Due to the existing complexity of the createtable command, the createtable help only
displays an example of setting one iterator upon table creation. It is
+   * possible to set multiple if needed. This test verifies that capability.
+   */
+  @Test
+  public void testCreateTableWithMultipleIterators() throws Exception {
+    final String tmpTable = "tmpTable";
+    final String table = name.getMethodName();
+
+    // create iterator profile
+    // Will use tmpTable for creating profile since setshelliter is requiring a table
+    // even though its command line help indicates that it is optional. Likely due to
+    // the fact that setshelliter extends setiter, which does require a table argument.
+    ts.exec("createtable " + tmpTable, true);
+    String output = ts.exec("tables");
+    Assert.assertTrue(output.contains(tmpTable));
+
+    ts.input.set("\n5000\n\n");
+    ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile1"));
+
+    ts.input.set("2\n");
+    ts.exec("setshelliter -n iter2 -p 11 -pn profile2 -vers -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile2"));
+
+    // create table making use of the iterator profiles
+    ts.exec("createtable " + table + " -i profile1:scan,minc profile2:all ", true);
+    ts.exec("insert foo a b c", true);
+    ts.exec("scan", true, "foo a:b []    c");
+    ts.exec("sleep 6", true);
+    ts.exec("scan", true, "", true);
+    output = ts.exec("listiter -t " + table + " -all");
+    Assert.assertTrue(output.contains("Iterator itname, scan scope options"));
+    Assert.assertTrue(output.contains("Iterator itname, minc scope options"));
+    Assert.assertFalse(output.contains("Iterator itname, majc scope options"));
+    Assert.assertTrue(output.contains("Iterator iter2, scan scope options"));
+    Assert.assertTrue(output.contains("Iterator iter2, minc scope options"));
+    Assert.assertTrue(output.contains("Iterator iter2, majc scope options"));
+    ts.exec("deletetable -f " + table);
+    ts.exec("deletetable -f " + tmpTable);
+  }
+
+  @Test
+  public void testCreateTableWithIteratorsBadArguments() throws IOException {
+    final String tmpTable = "tmpTable";
+    final String table = name.getMethodName();
+    ts.exec("createtable " + tmpTable, true);
+    String output = ts.exec("tables");
+    Assert.assertTrue(output.contains(tmpTable));
+    ts.input.set("\n5000\n\n");
+    ts.exec("setshelliter -n itname -p 10 -pn profile1 -ageoff -t " + tmpTable, true);
+    output = ts.exec("listshelliter");
+    Assert.assertTrue(output.contains("Profile : profile1"));
+    // test various bad argument calls
+    ts.exec("createtable " + table + " -i noprofile:scan,minc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,minc,all,majc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,all,majc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,min,majc", false);
+    ts.exec("createtable " + table + " -i profile1:scan,max,all", false);
+    ts.exec("createtable " + table + " -i profile1:", false);
+    ts.exec("createtable " + table + " -i profile1: ", false);
+    ts.exec("createtable " + table + " -i profile1:-scan", false);
+    ts.exec("createtable " + table + " profile1:majc", false);
+    ts.exec("createtable " + table + " -i profile1: all", false);
+    ts.exec("createtable " + table + " -i profile1: All", false);
+    ts.exec("createtable " + table + " -i profile1: scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc,Scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc, scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc,,scan", false);
+    ts.exec("createtable " + table + " -i profile1:minc,minc", false);
+    ts.exec("createtable " + table + " -i profile1:minc,Minc", false);
+    ts.exec("createtable " + table + " -i profile1:minc, ,scan", false);
+    ts.exec("createtable " + table + "-i", false);
+    ts.exec("createtable " + table + "-i ", false);
+    ts.exec("deletetable -f " + tmpTable);
+  }
 }

-- 
To stop receiving notification emails like this one, please contact
kturner@apache.org.

Mime
View raw message