accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ujustgotbi...@apache.org
Subject [49/50] git commit: ACCUMULO-564 changes for 0.23 compile compatibility
Date Thu, 06 Feb 2014 05:40:28 GMT
ACCUMULO-564 changes for 0.23 compile compatibility

git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk/examples/wikisearch@1332674 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 201ebbadb38ba5f71e870ea28c6e0120a027c8e3)

Reason: Hadoop2 Compat
Author: Billie Rinaldi <billie@apache.org>
Ref: ACCUMULO-1977


Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/c9f213e9
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/c9f213e9
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/c9f213e9

Branch: refs/heads/1.4.5-SNAPSHOT
Commit: c9f213e99defefce72146caf68f41ad0bf343aa7
Parents: 0c429f9
Author: Billie Rinaldi <billie@apache.org>
Authored: Tue May 1 14:35:39 2012 +0000
Committer: Sean Busbey <busbey@clouderagovt.com>
Committed: Tue Dec 10 14:06:40 2013 -0600

----------------------------------------------------------------------
 .../wikisearch/ingest/WikipediaIngester.java    |  10 +-
 .../ingest/StandaloneStatusReporter.java        |  70 --------
 .../wikisearch/ingest/WikipediaMapperTest.java  | 163 -------------------
 .../logic/StandaloneStatusReporter.java         |   4 +
 .../wikisearch/logic/TestQueryLogic.java        |  31 ++--
 5 files changed, 28 insertions(+), 250 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/c9f213e9/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
----------------------------------------------------------------------
diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
index 50415a7..d4fa1c6 100644
--- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
+++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
@@ -66,8 +66,8 @@ public class WikipediaIngester extends Configured implements Tool {
     System.exit(res);
   }
   
-  private void createTables(TableOperations tops, String tableName) throws AccumuloException,
AccumuloSecurityException, TableNotFoundException,
-      TableExistsException {
+  public static void createTables(TableOperations tops, String tableName, boolean configureLocalityGroups)
throws AccumuloException, AccumuloSecurityException,
+      TableNotFoundException, TableExistsException {
     // Create the shard table
     String indexTableName = tableName + "Index";
     String reverseIndexTableName = tableName + "ReverseIndex";
@@ -94,7 +94,9 @@ public class WikipediaIngester extends Configured implements Tool {
       }
       
       // Set the locality group for the full content column family
-      tops.setLocalityGroups(tableName, Collections.singletonMap("WikipediaDocuments", Collections.singleton(new
Text(WikipediaMapper.DOCUMENT_COLUMN_FAMILY))));
+      if (configureLocalityGroups)
+        tops.setLocalityGroups(tableName,
+            Collections.singletonMap("WikipediaDocuments", Collections.singleton(new Text(WikipediaMapper.DOCUMENT_COLUMN_FAMILY))));
       
     }
     
@@ -143,7 +145,7 @@ public class WikipediaIngester extends Configured implements Tool {
     
     TableOperations tops = connector.tableOperations();
     
-    createTables(tops, tablename);
+    createTables(tops, tablename, true);
     
     configureJob(job);
     

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/c9f213e9/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
----------------------------------------------------------------------
diff --git a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
b/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
deleted file mode 100644
index 6af1e9b..0000000
--- a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.StatusReporter;
-
-public class StandaloneStatusReporter extends StatusReporter {
-  
-  private Counters c = new Counters();
-  
-  private long filesProcessed = 0;
-  private long recordsProcessed = 0;
-  
-  public Counters getCounters() {
-    return c;
-  }
-  
-  @Override
-  public Counter getCounter(Enum<?> name) {
-    return c.findCounter(name);
-  }
-  
-  @Override
-  public Counter getCounter(String group, String name) {
-    return c.findCounter(group, name);
-  }
-  
-  @Override
-  public void progress() {
-    // do nothing
-  }
-  
-  @Override
-  public void setStatus(String status) {
-    // do nothing
-  }
-  
-  public long getFilesProcessed() {
-    return filesProcessed;
-  }
-  
-  public long getRecordsProcessed() {
-    return recordsProcessed;
-  }
-  
-  public void incrementFilesProcessed() {
-    filesProcessed++;
-    recordsProcessed = 0;
-  }
-  
-  public void incrementRecordsProcessed() {
-    recordsProcessed++;
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/c9f213e9/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
----------------------------------------------------------------------
diff --git a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
b/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
deleted file mode 100644
index c659ec4..0000000
--- a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import junit.framework.Assert;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.ContextFactory;
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-import org.junit.Before;
-
-/**
- * Load some data into mock accumulo
- */
-public class WikipediaMapperTest {
-  
-  private static final String METADATA_TABLE_NAME = "wikiMetadata";
-  
-  private static final String TABLE_NAME = "wiki";
-  
-  private static final String INDEX_TABLE_NAME = "wikiIndex";
-  
-  private static final String RINDEX_TABLE_NAME = "wikiReverseIndex";
-  
-  private class MockAccumuloRecordWriter extends RecordWriter<Text,Mutation> {
-    @Override
-    public void write(Text key, Mutation value) throws IOException, InterruptedException
{
-      try {
-        writerMap.get(key).addMutation(value);
-      } catch (MutationsRejectedException e) {
-        throw new IOException("Error adding mutation", e);
-      }
-    }
-    
-    @Override
-    public void close(TaskAttemptContext context) throws IOException, InterruptedException
{
-      try {
-        for (BatchWriter w : writerMap.values()) {
-          w.flush();
-          w.close();
-        }
-      } catch (MutationsRejectedException e) {
-        throw new IOException("Error closing Batch Writer", e);
-      }
-    }
-    
-  }
-  
-  private Connector c = null;
-  private Configuration conf = new Configuration();
-  private HashMap<Text,BatchWriter> writerMap = new HashMap<Text,BatchWriter>();
-  
-  @Before
-  public void setup() throws Exception {
-    
-    conf.set(AggregatingRecordReader.START_TOKEN, "<page>");
-    conf.set(AggregatingRecordReader.END_TOKEN, "</page>");
-    conf.set(WikipediaConfiguration.TABLE_NAME, TABLE_NAME);
-    conf.set(WikipediaConfiguration.NUM_PARTITIONS, "1");
-    conf.set(WikipediaConfiguration.NUM_GROUPS, "1");
-    
-    MockInstance i = new MockInstance();
-    c = i.getConnector("root", "pass");
-    c.tableOperations().delete(METADATA_TABLE_NAME);
-    c.tableOperations().delete(TABLE_NAME);
-    c.tableOperations().delete(INDEX_TABLE_NAME);
-    c.tableOperations().delete(RINDEX_TABLE_NAME);
-    c.tableOperations().create(METADATA_TABLE_NAME);
-    c.tableOperations().create(TABLE_NAME);
-    c.tableOperations().create(INDEX_TABLE_NAME);
-    c.tableOperations().create(RINDEX_TABLE_NAME);
-    
-    writerMap.put(new Text(METADATA_TABLE_NAME), c.createBatchWriter(METADATA_TABLE_NAME,
1000L, 1000L, 1));
-    writerMap.put(new Text(TABLE_NAME), c.createBatchWriter(TABLE_NAME, 1000L, 1000L, 1));
-    writerMap.put(new Text(INDEX_TABLE_NAME), c.createBatchWriter(INDEX_TABLE_NAME, 1000L,
1000L, 1));
-    writerMap.put(new Text(RINDEX_TABLE_NAME), c.createBatchWriter(RINDEX_TABLE_NAME, 1000L,
1000L, 1));
-    
-    TaskAttemptContext context = ContextFactory.createTaskAttemptContext(conf);
-    
-    RawLocalFileSystem fs = new RawLocalFileSystem();
-    fs.setConf(conf);
-    
-    URL url = ClassLoader.getSystemResource("enwiki-20110901-001.xml");
-    Assert.assertNotNull(url);
-    File data = new File(url.toURI());
-    Path tmpFile = new Path(data.getAbsolutePath());
-    
-    // Setup the Mapper
-    InputSplit split = new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null);
-    AggregatingRecordReader rr = new AggregatingRecordReader();
-    Path ocPath = new Path(tmpFile, "oc");
-    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
-    fs.deleteOnExit(ocPath);
-    StandaloneStatusReporter sr = new StandaloneStatusReporter();
-    rr.initialize(split, context);
-    MockAccumuloRecordWriter rw = new MockAccumuloRecordWriter();
-    WikipediaMapper mapper = new WikipediaMapper();
-    
-    // Load data into Mock Accumulo
-    Mapper<LongWritable,Text,Text,Mutation>.Context con = ContextFactory.createMapContext(mapper,
context, rr, rw, oc, sr, split);
-    mapper.run(con);
-    
-    // Flush and close record writers.
-    rw.close(context);
-    
-  }
-  
-  private void debugQuery(String tableName) throws Exception {
-    Scanner s = c.createScanner(tableName, new Authorizations("all"));
-    Range r = new Range();
-    s.setRange(r);
-    for (Entry<Key,Value> entry : s)
-      System.out.println(entry.getKey().toString() + " " + entry.getValue().toString());
-  }
-  
-  public void testViewAllData() throws Exception {
-    debugQuery(METADATA_TABLE_NAME);
-    debugQuery(TABLE_NAME);
-    debugQuery(INDEX_TABLE_NAME);
-    debugQuery(RINDEX_TABLE_NAME);
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/c9f213e9/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
b/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
index 35743b3..a3b90d2 100644
--- a/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
+++ b/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
@@ -67,4 +67,8 @@ public class StandaloneStatusReporter extends StatusReporter {
   public void incrementRecordsProcessed() {
     recordsProcessed++;
   }
+  
+  public float getProgress() {
+    return 0;
+  }
 }

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/c9f213e9/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
b/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
index 938f01b..9079d79 100644
--- a/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
+++ b/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
@@ -16,6 +16,8 @@
  */
 package org.apache.accumulo.examples.wikisearch.logic;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
@@ -38,12 +40,12 @@ import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.ContextFactory;
 import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration;
+import org.apache.accumulo.examples.wikisearch.ingest.WikipediaIngester;
 import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
 import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
 import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator;
 import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
 import org.apache.accumulo.examples.wikisearch.sample.Document;
-import org.apache.accumulo.examples.wikisearch.sample.Field;
 import org.apache.accumulo.examples.wikisearch.sample.Results;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -117,11 +119,8 @@ public class TestQueryLogic {
     
     MockInstance i = new MockInstance();
     c = i.getConnector("root", "");
+    WikipediaIngester.createTables(c.tableOperations(), TABLE_NAME, false);
     for (String table : TABLE_NAMES) {
-      try {
-        c.tableOperations().delete(table);
-      } catch (Exception ex) {}
-      c.tableOperations().create(table);
       writerMap.put(new Text(table), c.createBatchWriter(table, 1000L, 1000L, 1));
     }
     
@@ -162,7 +161,7 @@ public class TestQueryLogic {
   }
   
   void debugQuery(String tableName) throws Exception {
-    Scanner s = c.createScanner(tableName, new Authorizations());
+    Scanner s = c.createScanner(tableName, new Authorizations("all"));
     Range r = new Range();
     s.setRange(r);
     for (Entry<Key,Value> entry : s)
@@ -170,17 +169,23 @@ public class TestQueryLogic {
   }
   
   @Test
-  public void testTitle() {
+  public void testTitle() throws Exception {
     Logger.getLogger(AbstractQueryLogic.class).setLevel(Level.OFF);
     Logger.getLogger(RangeCalculator.class).setLevel(Level.OFF);
     List<String> auths = new ArrayList<String>();
     auths.add("enwiki");
-    Results results = table.runQuery(c, auths, "TITLE == 'afghanistanhistory'", null, null,
null);
-    for (Document doc : results.getResults()) {
-      System.out.println("id: " + doc.getId());
-      for (Field field : doc.getFields())
-        System.out.println(field.getFieldName() + " -> " + field.getFieldValue());
-    }
+    
+    Results results = table.runQuery(c, auths, "TITLE == 'asphalt' or TITLE == 'abacus' or
TITLE == 'acid' or TITLE == 'acronym'", null, null, null);
+    List<Document> docs = results.getResults();
+    assertEquals(4, docs.size());
+    
+    /*
+     * debugQuery(METADATA_TABLE_NAME); debugQuery(TABLE_NAME); debugQuery(INDEX_TABLE_NAME);
debugQuery(RINDEX_TABLE_NAME);
+     * 
+     * results = table.runQuery(c, auths, "TEXT == 'abacus'", null, null, null); docs = results.getResults();
assertEquals(4, docs.size()); for (Document doc :
+     * docs) { System.out.println("id: " + doc.getId()); for (Field field : doc.getFields())
System.out.println(field.getFieldName() + " -> " +
+     * field.getFieldValue()); }
+     */
   }
   
 }


Mime
View raw message