accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [24/53] [abbrv] ACCUMULO-658 Move tests and resources to correct modules
Date Fri, 06 Sep 2013 18:22:52 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/FindTablet.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/FindTablet.java
b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/FindTablet.java
new file mode 100644
index 0000000..f0a8268
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/FindTablet.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Finds tablet creation events.
+ */
+public class FindTablet {
+  
+  static public class Opts extends ClientOpts {
+    @Parameter(names = {"-r", "--row"}, required = true, description = "find tablets that
contain this row")
+    String row = null;
+    
+    @Parameter(names = "--tableId", required = true, description = "table id")
+    String tableId = null;
+  }
+  
+  public static void main(String[] args) throws Exception {
+    Opts opts = new Opts();
+    opts.parseArgs(FindTablet.class.getName(), args);
+    
+    findContainingTablets(opts);
+  }
+  
+  private static void findContainingTablets(Opts opts) throws Exception {
+    Range range = new KeyExtent(new Text(opts.tableId), null, null).toMetadataRange();
+    
+    Scanner scanner = opts.getConnector().createScanner("createEvents", opts.auths);
+    scanner.setRange(range);
+    
+    Text row = new Text(opts.row);
+    for (Entry<Key,Value> entry : scanner) {
+      KeyExtent ke = new KeyExtent(entry.getKey().getRow(), new Value(TextUtil.getBytes(entry.getKey().getColumnFamily())));
+      if (ke.contains(row)) {
+        System.out.println(entry.getKey().getColumnQualifier() + " " + ke + " " + entry.getValue());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/IndexMeta.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/IndexMeta.java
b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/IndexMeta.java
new file mode 100644
index 0000000..e42731a
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/IndexMeta.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.tserver.logger.LogEvents;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * A map reduce job that takes write ahead logs containing mutations for the metadata table
and indexes them into Accumulo tables for analysis.
+ * 
+ */
+
+public class IndexMeta extends Configured implements Tool {
+  
+  public static class IndexMapper extends Mapper<LogFileKey,LogFileValue,Text,Mutation>
{
+    private static final Text CREATE_EVENTS_TABLE = new Text("createEvents");
+    private static final Text TABLET_EVENTS_TABLE = new Text("tabletEvents");
+    private Map<Integer,KeyExtent> tabletIds = new HashMap<Integer,KeyExtent>();
+    private String uuid = null;
+    
+    @Override
+    protected void setup(Context context) throws java.io.IOException, java.lang.InterruptedException
{
+      tabletIds = new HashMap<Integer,KeyExtent>();
+      uuid = null;
+    }
+    
+    @Override
+    public void map(LogFileKey key, LogFileValue value, Context context) throws IOException,
InterruptedException {
+      if (key.event == LogEvents.OPEN) {
+        uuid = key.tserverSession;
+      } else if (key.event == LogEvents.DEFINE_TABLET) {
+        if (key.tablet.getTableId().toString().equals(MetadataTable.ID)) {
+          tabletIds.put(key.tid, new KeyExtent(key.tablet));
+        }
+      } else if ((key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS)
&& tabletIds.containsKey(key.tid)) {
+        for (Mutation m : value.mutations) {
+          index(context, m, uuid, tabletIds.get(key.tid));
+        }
+      }
+    }
+    
+    void index(Context context, Mutation m, String logFile, KeyExtent metaTablet) throws
IOException, InterruptedException {
+      List<ColumnUpdate> columnsUpdates = m.getUpdates();
+      
+      Text prevRow = null;
+      long timestamp = 0;
+      
+      if (m.getRow().length > 0 && m.getRow()[0] == '~') {
+        return;
+      }
+      
+      for (ColumnUpdate cu : columnsUpdates) {
+        if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()),
new Text(cu.getColumnQualifier())) && !cu.isDeleted()) {
+          prevRow = new Text(cu.getValue());
+        }
+        
+        timestamp = cu.getTimestamp();
+      }
+      
+      byte[] serMut = WritableUtils.toByteArray(m);
+      
+      if (prevRow != null) {
+        Mutation createEvent = new Mutation(new Text(m.getRow()));
+        createEvent.put(prevRow, new Text(String.format("%020d", timestamp)), new Value(metaTablet.toString().getBytes()));
+        context.write(CREATE_EVENTS_TABLE, createEvent);
+      }
+      
+      Mutation tabletEvent = new Mutation(new Text(m.getRow()));
+      tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("mut"), new Value(serMut));
+      tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("mtab"), new
Value(metaTablet.toString().getBytes()));
+      tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("log"), new Value(logFile.getBytes()));
+      context.write(TABLET_EVENTS_TABLE, tabletEvent);
+    }
+  }
+  
+  static class Opts extends ClientOpts {
+    @Parameter(description = "<logfile> { <logfile> ...}")
+    List<String> logFiles = new ArrayList<String>();
+  }
+  
+  @Override
+  public int run(String[] args) throws Exception {
+    Opts opts = new Opts();
+    opts.parseArgs(IndexMeta.class.getName(), args);
+    
+    String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();
+    
+    Job job = new Job(getConf(), jobName);
+    job.setJarByClass(this.getClass());
+    
+    List<String> logFiles = Arrays.asList(args).subList(4, args.length);
+    Path paths[] = new Path[logFiles.size()];
+    int count = 0;
+    for (String logFile : logFiles) {
+      paths[count++] = new Path(logFile);
+    }
+    
+    job.setInputFormatClass(LogFileInputFormat.class);
+    LogFileInputFormat.setInputPaths(job, paths);
+    
+    job.setNumReduceTasks(0);
+    
+    job.setOutputFormatClass(AccumuloOutputFormat.class);
+    AccumuloOutputFormat.setZooKeeperInstance(job, opts.instance, opts.zookeepers);
+    AccumuloOutputFormat.setConnectorInfo(job, opts.principal, opts.getToken());
+    AccumuloOutputFormat.setCreateTables(job, false);
+    
+    job.setMapperClass(IndexMapper.class);
+    
+    Connector conn = opts.getConnector();
+    
+    try {
+      conn.tableOperations().create("createEvents");
+    } catch (TableExistsException tee) {
+      Logger.getLogger(IndexMeta.class).warn("Table createEvents exists");
+    }
+    
+    try {
+      conn.tableOperations().create("tabletEvents");
+    } catch (TableExistsException tee) {
+      Logger.getLogger(IndexMeta.class).warn("Table tabletEvents exists");
+    }
+    
+    job.waitForCompletion(true);
+    return job.isSuccessful() ? 0 : 1;
+  }
+  
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(CachedConfiguration.getInstance(), new IndexMeta(), args);
+    System.exit(res);
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileInputFormat.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileInputFormat.java
b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileInputFormat.java
new file mode 100644
index 0000000..0b206ba
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileInputFormat.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+
+/**
+ * Input format for Accumulo write ahead logs
+ */
+public class LogFileInputFormat extends FileInputFormat<LogFileKey,LogFileValue> {
+  
+  private static class LogFileRecordReader extends RecordReader<LogFileKey,LogFileValue>
{
+    
+    private FSDataInputStream fsdis;
+    private LogFileKey key;
+    private LogFileValue value;
+    private long length;
+    
+    @Override
+    public void close() throws IOException {
+      fsdis.close();
+    }
+    
+    @Override
+    public LogFileKey getCurrentKey() throws IOException, InterruptedException {
+      return key;
+    }
+    
+    @Override
+    public LogFileValue getCurrentValue() throws IOException, InterruptedException {
+      return value;
+    }
+    
+    @Override
+    public float getProgress() throws IOException, InterruptedException {
+      float progress = (length - fsdis.getPos()) / (float) length;
+      if (progress < 0)
+        return 0;
+      return progress;
+    }
+    
+    @Override
+    public void initialize(InputSplit is, TaskAttemptContext context) throws IOException,
InterruptedException {
+      FileSplit fileSplit = (FileSplit) is;
+      
+      Configuration conf = new Configuration();
+      FileSystem fs = FileSystem.get(conf);
+      
+      key = new LogFileKey();
+      value = new LogFileValue();
+      
+      fsdis = fs.open(fileSplit.getPath());
+      FileStatus status = fs.getFileStatus(fileSplit.getPath());
+      length = status.getLen();
+    }
+
+    @Override
+    public boolean nextKeyValue() throws IOException, InterruptedException {
+      if (key == null)
+        return false;
+      
+      try {
+        key.readFields(fsdis);
+        value.readFields(fsdis);
+        return true;
+      } catch (EOFException ex) {
+        key = null;
+        value = null;
+        return false;
+      }
+    }
+    
+  }
+
+  
+  @Override
+  public RecordReader<LogFileKey,LogFileValue> createRecordReader(InputSplit arg0,
TaskAttemptContext arg1) throws IOException, InterruptedException {
+    return new LogFileRecordReader();
+  }
+  
+  @Override
+  protected boolean isSplitable(JobContext context, Path filename) {
+    return false;
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
new file mode 100644
index 0000000..f8dcc9e
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.IOException;
+
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+/**
+ * Output format for Accumulo write ahead logs.
+ */
+public class LogFileOutputFormat extends FileOutputFormat<LogFileKey,LogFileValue>
{
+  
+  private static class LogFileRecordWriter extends RecordWriter<LogFileKey,LogFileValue>
{
+    
+    private FSDataOutputStream out;
+    
+    public LogFileRecordWriter(Path outputPath) throws IOException {
+      Configuration conf = new Configuration();
+      FileSystem fs = FileSystem.get(conf);
+      
+      out = fs.create(outputPath);
+    }
+    
+    @Override
+    public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
+      out.close();
+    }
+    
+    @Override
+    public void write(LogFileKey key, LogFileValue val) throws IOException, InterruptedException
{
+      key.write(out);
+      val.write(out);
+    }
+    
+  }
+  
+  @Override
+  public RecordWriter<LogFileKey,LogFileValue> getRecordWriter(TaskAttemptContext context)
throws IOException, InterruptedException {
+    Path outputPath = getDefaultWorkFile(context, "");
+    return new LogFileRecordWriter(outputPath);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
new file mode 100644
index 0000000..499b6bd
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Looks up and prints mutations indexed by IndexMeta
+ */
+public class PrintEvents {
+  
+  static class Opts extends ClientOpts {
+    @Parameter(names = {"-t", "--tableId"}, description = "table id", required = true)
+    String tableId;
+    @Parameter(names = {"-e", "--endRow"}, description = "end row")
+    String endRow;
+    @Parameter(names = {"-t", "--time"}, description = "time, in milliseconds", required
= true)
+    long time;
+  }
+  
+  public static void main(String[] args) throws Exception {
+    Opts opts = new Opts();
+    opts.parseArgs(PrintEvents.class.getName(), args);
+    
+    Connector conn = opts.getConnector();
+    
+    printEvents(conn, opts.tableId, opts.endRow, opts.time);
+  }
+  
+  private static void printEvents(Connector conn, String tableId, String endRow, Long time)
throws Exception {
+    Scanner scanner = conn.createScanner("tabletEvents", new Authorizations());
+    String metaRow = tableId + (endRow == null ? "<" : ";" + endRow);
+    scanner.setRange(new Range(new Key(metaRow, String.format("%020d", time)), true, new
Key(metaRow).followingKey(PartialKey.ROW), false));
+    int count = 0;
+    
+    String lastLog = null;
+    
+    loop1: for (Entry<Key,Value> entry : scanner) {
+      if (entry.getKey().getColumnQualifier().toString().equals("log")) {
+        if (lastLog == null || !lastLog.equals(entry.getValue().toString()))
+          System.out.println("Log : " + entry.getValue());
+        lastLog = entry.getValue().toString();
+      } else if (entry.getKey().getColumnQualifier().toString().equals("mut")) {
+        DataInputStream dis = new DataInputStream(new ByteArrayInputStream(entry.getValue().get()));
+        Mutation m = new Mutation();
+        m.readFields(dis);
+        
+        LogFileValue lfv = new LogFileValue();
+        lfv.mutations = Collections.singletonList(m);
+        
+        System.out.println(LogFileValue.format(lfv, 1));
+        
+        List<ColumnUpdate> columnsUpdates = m.getUpdates();
+        for (ColumnUpdate cu : columnsUpdates) {
+          if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()),
new Text(cu.getColumnQualifier())) && count > 0) {
+            System.out.println("Saw change to prevrow, stopping printing events.");
+            break loop1;
+          }
+        }
+        count++;
+      }
+    }
+    
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/package-info.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/package-info.java
b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/package-info.java
new file mode 100644
index 0000000..4d404ed
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/metanalysis/package-info.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Provides programs to analyze metadata mutations written to write ahead logs.  
+ * 
+ * <p>
+ * These programs can be used when write ahead logs are archived.   The best way to find
+ * which write ahead logs contain metadata mutations is to grep the tablet server logs. 

+ * Grep for events where walogs were added to metadata tablets, then take the unique set

+ * of walogs.
+ *
+ * <p>
+ * To use these programs, use IndexMeta to index the metadata mutations in walogs into 
+ * Accumulo tables.  Then use FindTable and PrintEvents to analyze those indexes.  
+ * FilterMetaiallows filtering walogs down to just metadata events.  This is useful for the
+ * case where the walogs need to be exported from the cluster for analysis.
+ *
+ * @since 1.5
+ */
+package org.apache.accumulo.server.metanalysis;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/server/utils/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
----------------------------------------------------------------------
diff --git a/server/utils/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
b/server/utils/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
new file mode 100644
index 0000000..d7837b8
--- /dev/null
+++ b/server/utils/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.master.LiveTServerSet;
+import org.apache.accumulo.master.LiveTServerSet.Listener;
+import org.apache.accumulo.master.state.DistributedStoreException;
+import org.apache.accumulo.master.state.MetaDataTableScanner;
+import org.apache.accumulo.master.state.TServerInstance;
+import org.apache.accumulo.master.state.TabletLocationState;
+import org.apache.accumulo.master.state.TabletState;
+import org.apache.accumulo.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class FindOfflineTablets {
+  private static final Logger log = Logger.getLogger(FindOfflineTablets.class);
+
+  /**
+   * @param args
+   */
+  public static void main(String[] args) throws Exception {
+    ClientOpts opts = new ClientOpts();
+    opts.parseArgs(FindOfflineTablets.class.getName(), args);
+    Instance instance = opts.getInstance();
+    SystemCredentials creds = SystemCredentials.get();
+
+    findOffline(instance, creds, null);
+  }
+
+  static int findOffline(Instance instance, Credentials creds, String tableName) throws AccumuloException,
TableNotFoundException {
+
+    final AtomicBoolean scanning = new AtomicBoolean(false);
+
+    LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(),
new Listener() {
+      @Override
+      public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance>
added) {
+        if (!deleted.isEmpty() && scanning.get())
+          log.warn("Tablet servers deleted while scanning: " + deleted);
+        if (!added.isEmpty() && scanning.get())
+          log.warn("Tablet servers added while scanning: " + added);
+      }
+    });
+    tservers.startListeningForTabletServerChanges();
+    scanning.set(true);
+
+    Iterator<TabletLocationState> zooScanner;
+    try {
+      zooScanner = new ZooTabletStateStore().iterator();
+    } catch (DistributedStoreException e) {
+      throw new AccumuloException(e);
+    }
+
+    int offline = 0;
+
+    System.out.println("Scanning zookeeper");
+    if ((offline = checkTablets(zooScanner, tservers)) > 0)
+      return offline;
+
+    if (RootTable.NAME.equals(tableName))
+      return 0;
+
+    System.out.println("Scanning " + RootTable.NAME);
+    Iterator<TabletLocationState> rootScanner = new MetaDataTableScanner(instance,
creds, MetadataSchema.TabletsSection.getRange(), RootTable.NAME);
+    if ((offline = checkTablets(rootScanner, tservers)) > 0)
+      return offline;
+
+    if (MetadataTable.NAME.equals(tableName))
+      return 0;
+
+    System.out.println("Scanning " + MetadataTable.NAME);
+
+    Range range = MetadataSchema.TabletsSection.getRange();
+    if (tableName != null) {
+      String tableId = Tables.getTableId(instance, tableName);
+      range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+    }
+
+    Iterator<TabletLocationState> metaScanner = new MetaDataTableScanner(instance,
creds, range, MetadataTable.NAME);
+    return checkTablets(metaScanner, tservers);
+  }
+
+  private static int checkTablets(Iterator<TabletLocationState> scanner, LiveTServerSet
tservers) {
+    int offline = 0;
+
+    while (scanner.hasNext()) {
+      TabletLocationState locationState = scanner.next();
+      TabletState state = locationState.getState(tservers.getCurrentServers());
+      if (state != null && state != TabletState.HOSTED
+          && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString())
!= TableState.OFFLINE) {
+        System.out.println(locationState + " is " + state + "  #walogs:" + locationState.walogs.size());
+        offline++;
+      }
+    }
+
+    return offline;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index 8343bb2..34fae7f 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -43,11 +43,15 @@
     </dependency>
     <dependency>
       <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-gc</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
       <artifactId>accumulo-minicluster</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.accumulo</groupId>
-      <artifactId>accumulo-server</artifactId>
+      <artifactId>accumulo-monitor</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.accumulo</groupId>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/test/src/main/resources/randomwalk/Basic.xml
----------------------------------------------------------------------
diff --git a/test/src/main/resources/randomwalk/Basic.xml b/test/src/main/resources/randomwalk/Basic.xml
new file mode 100644
index 0000000..2dead02
--- /dev/null
+++ b/test/src/main/resources/randomwalk/Basic.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+
+<init id="test.CreateTable"/>
+
+<node id="test.CreateTable">
+  <edge id="unit/Simple.xml" weight="1"/>
+</node>
+
+<node id="unit/Simple.xml">
+  <edge id="unit/Simple.xml" weight="3"/>
+  <edge id="test.DeleteTable" weight="1"/>
+</node>
+
+<node id="test.DeleteTable">
+  <edge id="END" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/test/src/main/resources/randomwalk/Simple.xml
----------------------------------------------------------------------
diff --git a/test/src/main/resources/randomwalk/Simple.xml b/test/src/main/resources/randomwalk/Simple.xml
new file mode 100644
index 0000000..cad940e
--- /dev/null
+++ b/test/src/main/resources/randomwalk/Simple.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+
+<init id="dummy.all"/>
+
+<node id="dummy.all">
+  <edge id="test.Ingest" weight="1"/>
+  <edge id="test.Verify" weight="1"/>
+  <edge id="test.Scan" weight="1"/>
+  <edge id="END" weight="1"/>
+</node>
+
+<node id="test.Ingest">
+  <edge id="dummy.all" weight="1"/>
+</node>
+
+<node id="test.Verify">
+  <edge id="dummy.all" weight="1"/>
+</node>
+
+<node id="test.Scan">
+  <edge id="dummy.all" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ba20cf3/test/src/main/resources/randomwalk/module.xsd
----------------------------------------------------------------------
diff --git a/test/src/main/resources/randomwalk/module.xsd b/test/src/main/resources/randomwalk/module.xsd
new file mode 100644
index 0000000..bcdaaae0
--- /dev/null
+++ b/test/src/main/resources/randomwalk/module.xsd
@@ -0,0 +1,69 @@
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+  <xsd:element name="module" type="ModuleType"/>
+
+  <xsd:complexType name="ModuleType">
+    <xsd:sequence>
+      <xsd:element name="package" type="PrefixType" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="fixture" type="InitType" minOccurs="0" maxOccurs="1"/>
+      <xsd:element name="init" type="InitType"/>
+      <xsd:element name="node" type="NodeType" minOccurs="1" maxOccurs="unbounded"/>
+   </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="PrefixType">
+    <xsd:attribute name="prefix" type="xsd:string"/>
+    <xsd:attribute name="value" type="xsd:string"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="InitType">
+    <xsd:attribute name="id" type="xsd:string"/>
+    <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="teardown" type="xsd:boolean"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="NodeType">
+    <xsd:sequence>
+      <xsd:element name="alias" type="AliasType" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="edge" type="EdgeType" minOccurs="1" maxOccurs="unbounded"/>
+    </xsd:sequence>
+    <xsd:attribute name="id" type="xsd:string"/>
+    <xsd:attribute name="src" type="xsd:string"/>
+    <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="teardown" type="xsd:boolean"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="EdgeType">
+    <xsd:attribute name="id" type="xsd:string"/>
+    <xsd:attribute name="weight" type="xsd:positiveInteger"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="AliasType">
+    <xsd:attribute name="name" type="xsd:string"/>
+  </xsd:complexType>
+  
+  <xsd:complexType name="PropertyType">
+    <xsd:attribute name="key" type="xsd:string"/>
+    <xsd:attribute name="value" type="xsd:string"/>
+  </xsd:complexType>
+
+</xsd:schema>


Mime
View raw message