accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1412374 - in /accumulo/trunk/examples/instamo: ./ src/ src/main/ src/main/java/ src/main/java/org/ src/main/java/org/apache/ src/main/java/org/apache/accumulo/ src/main/java/org/apache/accumulo/instamo/ src/main/resources/ src/test/ src/te...
Date Thu, 22 Nov 2012 00:03:44 GMT
Author: kturner
Date: Thu Nov 22 00:03:42 2012
New Revision: 1412374

URL: http://svn.apache.org/viewvc?rev=1412374&view=rev
Log:
ACCUMULO-14 added mini cluster example

Added:
    accumulo/trunk/examples/instamo/
    accumulo/trunk/examples/instamo/README.md
    accumulo/trunk/examples/instamo/pom.xml
    accumulo/trunk/examples/instamo/src/
    accumulo/trunk/examples/instamo/src/main/
    accumulo/trunk/examples/instamo/src/main/java/
    accumulo/trunk/examples/instamo/src/main/java/org/
    accumulo/trunk/examples/instamo/src/main/java/org/apache/
    accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/
    accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/
    accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
  (with props)
    accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
  (with props)
    accumulo/trunk/examples/instamo/src/main/resources/
    accumulo/trunk/examples/instamo/src/main/resources/log4j.properties
    accumulo/trunk/examples/instamo/src/test/
    accumulo/trunk/examples/instamo/src/test/java/
    accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java   (with props)

Added: accumulo/trunk/examples/instamo/README.md
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/README.md?rev=1412374&view=auto
==============================================================================
--- accumulo/trunk/examples/instamo/README.md (added)
+++ accumulo/trunk/examples/instamo/README.md Thu Nov 22 00:03:42 2012
@@ -0,0 +1,27 @@
+Instamo
+=======
+
+Introduction
+-----------
+
+Instamo makes it easy to write some code and run it against a local, transient
+[Accumulo](http://accumulo.apache.org) instance in minutes.  No setup or
+installation is required.  This is possible if Java and Maven are already
+installed by following the steps below.
+
+```
+vim src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
+mvn package
+```
+
+Map Reduce
+----------
+
+Its possible to run local map reduce jobs against the MiniAccumuloCluster
+instance.   There is an example of this in the src directory  The following
+command will run the map reduce example.
+
+```
+mvn exec:exec
+```
+

Added: accumulo/trunk/examples/instamo/pom.xml
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/pom.xml?rev=1412374&view=auto
==============================================================================
--- accumulo/trunk/examples/instamo/pom.xml (added)
+++ accumulo/trunk/examples/instamo/pom.xml Thu Nov 22 00:03:42 2012
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.accumulo</groupId>
+  <artifactId>instamo</artifactId>
+  <packaging>jar</packaging>
+  <version>1.5.0-SNAPSHOT</version>
+  <name>instamo</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <accumulo.version>1.5.0-SNAPSHOT</accumulo.version>
+    <maclass>org.apache.accumulo.instamo.MapReduceExample</maclass>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.11</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-core</artifactId>
+      <version>${accumulo.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-server</artifactId>
+      <version>${accumulo.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>3.3.4</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-core</artifactId>
+      <version>1.0.1</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.4</version>
+    </dependency>
+    <dependency>
+    	<groupId>org.apache.accumulo</groupId>
+    	<artifactId>accumulo-test</artifactId>
+    	<version>${accumulo.version}</version>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.0.2</version>
+        <configuration>
+          <source>1.6</source>
+          <target>1.6</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <configuration>
+          <executable>java</executable>
+          <arguments>
+            <argument>-classpath</argument>
+            <classpath/>
+            <argument>${maclass}</argument>
+          </arguments>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

Added: accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java?rev=1412374&view=auto
==============================================================================
--- accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
(added)
+++ accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
Thu Nov 22 00:03:42 2012
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.instamo;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+
+public class AccumuloApp {
+  
+  public static void run(String instanceName, String zookeepers, String rootPassword, String
args[]) throws Exception {
+    // edit this method to play with Accumulo
+
+    Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
+    
+    Connector conn = instance.getConnector("root", rootPassword);
+    
+    conn.tableOperations().create("foo");
+    
+    BatchWriter bw = conn.createBatchWriter("foo", new BatchWriterConfig());
+    Mutation m = new Mutation("r1");
+    m.put("cf1", "cq1", "v1");
+    m.put("cf1", "cq2", "v3");
+    bw.addMutation(m);
+    bw.close();
+    
+    Scanner scanner = conn.createScanner("foo", Constants.NO_AUTHS);
+    for (Entry<Key,Value> entry : scanner) {
+      System.out.println(entry.getKey() + " " + entry.getValue());
+    }
+    
+    conn.tableOperations().delete("foo");
+  }
+}

Propchange: accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/AccumuloApp.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java?rev=1412374&view=auto
==============================================================================
--- accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
(added)
+++ accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
Thu Nov 22 00:03:42 2012
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.instamo;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.UUID;
+
+import org.apache.accumulo.server.test.continuous.ContinuousIngest;
+import org.apache.accumulo.server.test.continuous.ContinuousVerify;
+import org.apache.accumulo.test.MiniAccumuloCluster;
+import org.apache.commons.io.FileUtils;
+
+/**
+ * An example of running local map reduce against MiniAccumuloCluster
+ */
+public class MapReduceExample {
+  
+  public static void run(String instanceName, String zookeepers, String rootPassword, String
args[]) throws Exception {
+    
+    // run continuous ingest to create data. This is not a map reduce job
+    ContinuousIngest.main(new String[] {instanceName, zookeepers, "root", rootPassword, "ci",
"5000000", "0", Long.MAX_VALUE + "", "1000", "1000", "1000000",
+        "60000", "2", "false"});
+    
+    String outputDir = FileUtils.getTempDirectoryPath() + File.separator + "ci_verify" +
UUID.randomUUID().toString();
+    
+    try {
+      // run verify map reduce job locally. This jobs looks for holes in the linked list
create by continuous ingest
+      ContinuousVerify.main(new String[] {"-D", "mapred.job.tracker=local", "-D", "fs.default.name=file:///",
instanceName, zookeepers, "root", rootPassword,
+          "ci", outputDir, "4", "1", "false"});
+    } finally {
+      // delete output dir of mapreduce job
+      FileUtils.deleteQuietly(new File(outputDir));
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    File tmpDir = new File(FileUtils.getTempDirectory(), "macc-" + UUID.randomUUID().toString());
+    
+    try {
+      MiniAccumuloCluster la = new MiniAccumuloCluster(tmpDir, "pass1234", new HashMap<String,String>());
+      la.start();
+      
+      System.out.println("\n   ---- Running Mapred Against Accumulo\n");
+      
+      run(la.getInstanceName(), la.getZookeepers(), "pass1234", args);
+      
+      System.out.println("\n   ---- Ran Mapred Against Accumulo\n");
+      
+      la.stop();
+    } finally {
+      FileUtils.deleteQuietly(tmpDir);
+    }
+  }
+}

Propchange: accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: accumulo/trunk/examples/instamo/src/main/resources/log4j.properties
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/main/resources/log4j.properties?rev=1412374&view=auto
==============================================================================
--- accumulo/trunk/examples/instamo/src/main/resources/log4j.properties (added)
+++ accumulo/trunk/examples/instamo/src/main/resources/log4j.properties Thu Nov 22 00:03:42
2012
@@ -0,0 +1,9 @@
+log4j.rootLogger=INFO, CA
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
+
+log4j.logger.org.apache.zookeeper=ERROR,CA
+log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
+log4j.logger.org.apache.accumulo.server.security.Auditor=off
+

Added: accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java?rev=1412374&view=auto
==============================================================================
--- accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java (added)
+++ accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java Thu Nov 22
00:03:42 2012
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.util.HashMap;
+
+import org.apache.accumulo.instamo.AccumuloApp;
+import org.apache.accumulo.test.MiniAccumuloCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * An example unit test that shows how to use MiniAccumuloCluster in a unit test
+ */
+
+public class ExampleAccumuloUnitTest {
+  
+  public static TemporaryFolder folder = new TemporaryFolder();
+  
+  private static MiniAccumuloCluster accumulo;
+
+  @BeforeClass
+  public static void setupMiniCluster() throws Exception {
+
+    folder.create();
+    
+    accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret", new HashMap<String,String>());
+    
+    accumulo.start();
+    
+  }
+
+  @Test(timeout = 30000)
+  public void test() throws Exception {
+    AccumuloApp.run(accumulo.getInstanceName(), accumulo.getZookeepers(), "superSecret",
new String[0]);
+  }
+  
+  @AfterClass
+  public static void tearDownMiniCluster() throws Exception {
+    accumulo.stop();
+    folder.delete();
+  }
+  
+}

Propchange: accumulo/trunk/examples/instamo/src/test/java/ExampleAccumuloUnitTest.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message