incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/5] git commit: Adding a mapred common library. This should allow a single project for the mapred code. The next step will be to remove the blur-mapred-hadoop1 and 2 projects.
Date Sun, 12 Apr 2015 13:26:35 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 8008efeae -> e8165b77a


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
deleted file mode 100644
index 5779d47..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
+++ /dev/null
@@ -1,377 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.mapreduce.lib;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.blur.analysis.FieldManager;
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.blur.lucene.LuceneVersionConstant;
-import org.apache.blur.lucene.codec.Blur024Codec;
-import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE;
-import org.apache.blur.server.TableContext;
-import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.Record;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.RowDocumentUtil;
-import org.apache.blur.utils.ShardUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.util.Progressable;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.lucene.store.SimpleFSDirectory;
-
-public class GenericBlurRecordWriter {
-
-	  private static final Log LOG = LogFactory.getLog(GenericBlurRecordWriter.class);
-	  private static final String JAVA_IO_TMPDIR = "java.io.tmpdir";
-	  private static final Counter NULL_COUNTER = new NullCounter();
-
-	  private final Text _prevKey = new Text();
-
-	  private final IndexWriter _writer;
-	  private final FieldManager _fieldManager;
-	  private final Directory _finalDir;
-	  private final Directory _localDir;
-	  private final File _localPath;
-	  // private final int _maxDocumentBufferSize;
-	  private final IndexWriterConfig _conf;
-	  private final IndexWriterConfig _overFlowConf;
-	  private final Path _newIndex;
-	  private final boolean _indexLocally;
-	  private final boolean _optimizeInFlight;
-	  private final DocumentBufferStrategy _documentBufferStrategy;
-	  private Counter _columnCount = NULL_COUNTER;
-	  private Counter _fieldCount = NULL_COUNTER;
-	  private Counter _recordCount = NULL_COUNTER;
-	  private Counter _rowCount = NULL_COUNTER;
-	  private Counter _recordDuplicateCount = NULL_COUNTER;
-	  private Counter _rowOverFlowCount = NULL_COUNTER;
-	  private Counter _rowDeleteCount = NULL_COUNTER;
-	  private RateCounter _recordRateCounter = new RateCounter(NULL_COUNTER);
-	  private RateCounter _rowRateCounter = new RateCounter(NULL_COUNTER);
-	  private RateCounter _copyRateCounter = new RateCounter(NULL_COUNTER);
-	  private boolean _countersSetup = false;
-	  private IndexWriter _localTmpWriter;
-	  private boolean _usingLocalTmpindex;
-	  private File _localTmpPath;
-	  private ProgressableDirectory _localTmpDir;
-	  private String _deletedRowId;
-	  private Configuration _configuration;
-
-	  public GenericBlurRecordWriter(Configuration configuration, int attemptId, String tmpDirName)
throws IOException {
-	    _configuration = configuration;
-	    _documentBufferStrategy = BlurOutputFormat.getDocumentBufferStrategy(_configuration);
-	    _indexLocally = BlurOutputFormat.isIndexLocally(_configuration);
-	    _optimizeInFlight = BlurOutputFormat.isOptimizeInFlight(_configuration);
-
-	    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(_configuration);
-	    int shardCount = tableDescriptor.getShardCount();
-	    int shardId = attemptId % shardCount;
-
-	    Path tableOutput = BlurOutputFormat.getOutputPath(_configuration);
-	    String shardName = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId);
-	    Path indexPath = new Path(tableOutput, shardName);
-	    _newIndex = new Path(indexPath, tmpDirName);
-	    _finalDir = new ProgressableDirectory(new HdfsDirectory(_configuration, _newIndex),
getProgressable());
-	    _finalDir.setLockFactory(NoLockFactory.getNoLockFactory());
-
-	    TableContext tableContext = TableContext.create(tableDescriptor);
-	    _fieldManager = tableContext.getFieldManager();
-	    Analyzer analyzer = _fieldManager.getAnalyzerForIndex();
-
-	    _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
-	    _conf.setCodec(new Blur024Codec());
-	    _conf.setSimilarity(tableContext.getSimilarity());
-	    TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
-	    mergePolicy.setUseCompoundFile(false);
-
-	    _overFlowConf = _conf.clone();
-
-	    if (_indexLocally) {
-	      String localDirPath = System.getProperty(JAVA_IO_TMPDIR);
-	      _localPath = new File(localDirPath, UUID.randomUUID().toString() + ".tmp");
-	      SimpleFSDirectory directory = new SimpleFSDirectory(_localPath);
-	      _localDir = new ProgressableDirectory(directory, getProgressable());
-	      _writer = new IndexWriter(_localDir, _conf.clone());
-	    } else {
-	      _localPath = null;
-	      _localDir = null;
-	      _writer = new IndexWriter(_finalDir, _conf.clone());
-	    }
-	  }
-
-	  private Progressable getProgressable() {
-	    final Progressable prg = BlurOutputFormat.getProgressable();
-	    return new Progressable() {
-
-	      private Progressable _progressable = prg;
-	      private long _lastWarn = 0;
-	      private boolean _progressSetupLogged = false;
-
-	      @Override
-	      public void progress() {
-	        if (_progressable != null) {
-	          _progressable.progress();
-	          if (!_progressSetupLogged) {
-	            LOG.info("Progress has been setup correctly.");
-	            _progressSetupLogged = true;
-	          }
-	        } else {
-	          Progressable progressable = BlurOutputFormat.getProgressable();
-	          if (progressable != null) {
-	            _progressable = progressable;
-	          } else {
-	            long now = System.nanoTime();
-	            if (_lastWarn + TimeUnit.SECONDS.toNanos(10) < now) {
-	              LOG.warn("Progress not being reported.");
-	              _lastWarn = System.nanoTime();
-	            }
-	          }
-	        }
-	      }
-	    };
-	  }
-
-	  public void write(Text key, BlurMutate value) throws IOException {
-	    if (!_countersSetup) {
-	      setupCounter();
-	      _countersSetup = true;
-	    }
-	    if (!_prevKey.equals(key)) {
-	      flush();
-	      _prevKey.set(key);
-	    }
-	    add(value);
-	  }
-
-	  private void setupCounter() {
-	    GetCounter getCounter = BlurOutputFormat.getGetCounter();
-	    if (getCounter != null) {
-	      _fieldCount = getCounter.getCounter(BlurCounters.LUCENE_FIELD_COUNT);
-	      _columnCount = getCounter.getCounter(BlurCounters.COLUMN_COUNT);
-	      _recordCount = getCounter.getCounter(BlurCounters.RECORD_COUNT);
-	      _recordDuplicateCount = getCounter.getCounter(BlurCounters.RECORD_DUPLICATE_COUNT);
-	      _rowCount = getCounter.getCounter(BlurCounters.ROW_COUNT);
-	      _rowDeleteCount = getCounter.getCounter(BlurCounters.ROW_DELETE_COUNT);
-	      _rowOverFlowCount = getCounter.getCounter(BlurCounters.ROW_OVERFLOW_COUNT);
-	      _recordRateCounter = new RateCounter(getCounter.getCounter(BlurCounters.RECORD_RATE));
-	      _rowRateCounter = new RateCounter(getCounter.getCounter(BlurCounters.ROW_RATE));
-	      _copyRateCounter = new RateCounter(getCounter.getCounter(BlurCounters.COPY_RATE));
-	    }
-	  }
-
-	  private void add(BlurMutate value) throws IOException {
-	    BlurRecord blurRecord = value.getRecord();
-	    Record record = getRecord(blurRecord);
-	    String recordId = record.getRecordId();
-	    if (value.getMutateType() == MUTATE_TYPE.DELETE) {
-	      _deletedRowId = blurRecord.getRowId();
-	      return;
-	    }
-	    if (_countersSetup) {
-	      _columnCount.increment(record.getColumns().size());
-	    }
-	    List<Field> document = RowDocumentUtil.getDoc(_fieldManager, blurRecord.getRowId(),
record);
-	    List<Field> dup = _documentBufferStrategy.add(recordId, document);
-	    if (_countersSetup) {
-	      if (dup != null) {
-	        _recordDuplicateCount.increment(1);
-	      } else {
-	        _fieldCount.increment(document.size());
-	        _recordCount.increment(1);
-	      }
-	    }
-	    flushToTmpIndexIfNeeded();
-	  }
-
-	  private void flushToTmpIndexIfNeeded() throws IOException {
-	    if (_documentBufferStrategy.isFull()) {
-	      LOG.info("Document Buffer is full overflow to disk.");
-	      flushToTmpIndex();
-	    }
-	  }
-
-	  private void flushToTmpIndex() throws IOException {
-	    if (_documentBufferStrategy.isEmpty()) {
-	      return;
-	    }
-	    _usingLocalTmpindex = true;
-	    if (_localTmpWriter == null) {
-	      String localDirPath = System.getProperty(JAVA_IO_TMPDIR);
-	      _localTmpPath = new File(localDirPath, UUID.randomUUID().toString() + ".tmp");
-	      SimpleFSDirectory directory = new SimpleFSDirectory(_localTmpPath);
-	      _localTmpDir = new ProgressableDirectory(directory, getProgressable());
-	      _localTmpWriter = new IndexWriter(_localTmpDir, _overFlowConf.clone());
-	      // The local tmp writer has merging disabled so the first document in is
-	      // going to be doc 0.
-	      // Therefore the first document added is the prime doc
-	      List<List<Field>> docs = _documentBufferStrategy.getAndClearBuffer();
-	      for (List<Field> doc : docs) {
-	        _localTmpWriter.addDocument(doc);
-	      }
-	    } else {
-	      List<List<Field>> docs = _documentBufferStrategy.getAndClearBuffer();
-	      for (List<Field> doc : docs) {
-	        _localTmpWriter.addDocument(doc);
-	      }
-	    }
-	  }
-
-	  private void resetLocalTmp() {
-	    _usingLocalTmpindex = false;
-	    _localTmpWriter = null;
-	    _localTmpDir = null;
-	    rm(_localTmpPath);
-	    _localTmpPath = null;
-	  }
-
-	  public static Record getRecord(BlurRecord value) {
-	    Record record = new Record();
-	    record.setRecordId(value.getRecordId());
-	    record.setFamily(value.getFamily());
-	    for (BlurColumn col : value.getColumns()) {
-	      record.addToColumns(new Column(col.getName(), col.getValue()));
-	    }
-	    return record;
-	  }
-
-	  private void flush() throws CorruptIndexException, IOException {
-	    if (_usingLocalTmpindex) {
-	      // since we have flushed to disk then we do not need to index the
-	      // delete.
-	      flushToTmpIndex();
-	      _localTmpWriter.close(false);
-	      DirectoryReader reader = DirectoryReader.open(_localTmpDir);
-	      AtomicReader atomicReader = SlowCompositeReaderWrapper.wrap(reader);
-	      AtomicReader primeDocAtomicReader= PrimeDocOverFlowHelper.addPrimeDoc(atomicReader);
-	      if (_countersSetup) {
-	        _recordRateCounter.mark(reader.numDocs());
-	      }
-	      _writer.addIndexes(primeDocAtomicReader);
-	      primeDocAtomicReader.close();
-	      resetLocalTmp();
-	      _writer.maybeMerge();
-	      if (_countersSetup) {
-	        _rowOverFlowCount.increment(1);
-	      }
-	    } else {
-	      if (_documentBufferStrategy.isEmpty()) {
-	        if (_deletedRowId != null) {
-	          _writer.addDocument(getDeleteDoc());
-	          if (_countersSetup) {
-	            _rowDeleteCount.increment(1);
-	          }
-	        }
-	      } else {
-	        List<List<Field>> docs = _documentBufferStrategy.getAndClearBuffer();
-	        docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
-	        _writer.addDocuments(docs);
-	        if (_countersSetup) {
-	          _recordRateCounter.mark(docs.size());
-	        }
-	      }
-	    }
-	    _deletedRowId = null;
-	    if (_countersSetup) {
-	      _rowRateCounter.mark();
-	      _rowCount.increment(1);
-	    }
-	  }
-
-	  private Document getDeleteDoc() {
-	    Document document = new Document();
-	    document.add(new StringField(BlurConstants.ROW_ID, _deletedRowId, Store.NO));
-	    document.add(new StringField(BlurConstants.DELETE_MARKER, BlurConstants.DELETE_MARKER_VALUE,
Store.NO));
-	    return document;
-	  }
-
-	  public void close() throws IOException {
-	    flush();
-	    _writer.close();
-	    if (_countersSetup) {
-	      _recordRateCounter.close();
-	      _rowRateCounter.close();
-	    }
-	    if (_indexLocally) {
-	      if (_optimizeInFlight) {
-	        copyAndOptimizeInFlightDir();
-	      } else {
-	        copyDir();
-	      }
-	    }
-	    if (_countersSetup) {
-	      _copyRateCounter.close();
-	    }
-	  }
-
-	  private void copyAndOptimizeInFlightDir() throws IOException {
-	    CopyRateDirectory copyRateDirectory = new CopyRateDirectory(_finalDir, _copyRateCounter);
-	    copyRateDirectory.setLockFactory(NoLockFactory.getNoLockFactory());
-	    DirectoryReader reader = DirectoryReader.open(_localDir);
-	    IndexWriter writer = new IndexWriter(copyRateDirectory, _conf.clone());
-	    writer.addIndexes(reader);
-	    writer.close();
-	    rm(_localPath);
-	  }
-
-	  private void copyDir() throws IOException {
-	    CopyRateDirectory copyRateDirectory = new CopyRateDirectory(_finalDir, _copyRateCounter);
-	    String[] fileNames = _localDir.listAll();
-	    for (String fileName : fileNames) {
-	      LOG.info("Copying [{0}] to [{1}]", fileName, _newIndex);
-	      _localDir.copy(copyRateDirectory, fileName, fileName, IOContext.DEFAULT);
-	    }
-	    rm(_localPath);
-	  }
-
-	  private void rm(File file) {
-	    if (!file.exists()) {
-	      return;
-	    }
-	    if (file.isDirectory()) {
-	      for (File f : file.listFiles()) {
-	        rm(f);
-	      }
-	    }
-	    file.delete();
-	  }
-
-	}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GetCounter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GetCounter.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GetCounter.java
deleted file mode 100644
index 6814ac2..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GetCounter.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-import org.apache.hadoop.mapreduce.Counter;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * 
- */
-public interface GetCounter {
-  
-  Counter getCounter(Enum<?> counterName);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
deleted file mode 100644
index 46c030f..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-public class IOUtil {
-
-  public static final String UTF_8 = "UTF-8";
-
-  public static String readString(DataInput input) throws IOException {
-    int length = readVInt(input);
-    byte[] buffer = new byte[length];
-    input.readFully(buffer);
-    return new String(buffer, UTF_8);
-  }
-
-  public static void writeString(DataOutput output, String s) throws IOException {
-    byte[] bs = s.getBytes(UTF_8);
-    writeVInt(output, bs.length);
-    output.write(bs);
-  }
-
-  public static int readVInt(DataInput input) throws IOException {
-    byte b = input.readByte();
-    int i = b & 0x7F;
-    for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-      b = input.readByte();
-      i |= (b & 0x7F) << shift;
-    }
-    return i;
-  }
-
-  public static void writeVInt(DataOutput output, int i) throws IOException {
-    while ((i & ~0x7F) != 0) {
-      output.writeByte((byte) ((i & 0x7f) | 0x80));
-      i >>>= 7;
-    }
-    output.writeByte((byte) i);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java
deleted file mode 100644
index 4b180fd..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.mapreduce.lib;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.mapreduce.Counter;
-
-public class NullCounter implements Counter {
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-
-  }
-
-  @Override
-  public void setDisplayName(String displayName) {
-
-  }
-
-  @Override
-  public String getName() {
-    return null;
-  }
-
-  @Override
-  public String getDisplayName() {
-    return null;
-  }
-
-  @Override
-  public long getValue() {
-    return 0;
-  }
-
-  @Override
-  public void setValue(long value) {
-
-  }
-
-  @Override
-  public void increment(long incr) {
-
-  }
-
-  @Override
-  public Counter getUnderlyingCounter() {
-    return null;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/PrimeDocOverFlowHelper.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/PrimeDocOverFlowHelper.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/PrimeDocOverFlowHelper.java
deleted file mode 100644
index 672a1c1..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/PrimeDocOverFlowHelper.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.mapreduce.lib;
-
-import java.io.IOException;
-
-import org.apache.blur.utils.BlurConstants;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FilterAtomicReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.ParallelAtomicReader;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.Version;
-
-public class PrimeDocOverFlowHelper {
-
-  private static Directory _directory;
-
-  static {
-    try {
-      _directory = new RAMDirectory();
-      IndexWriter writer = new IndexWriter(_directory, new IndexWriterConfig(Version.LUCENE_43,
new KeywordAnalyzer()));
-      Document document = new Document();
-      document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
-      writer.addDocument(document);
-      writer.close();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  public static AtomicReader addPrimeDoc(AtomicReader atomicReader) throws IOException {
-    AtomicReaderContext context = DirectoryReader.open(_directory).leaves().get(0);
-    return new ParallelAtomicReader(true, setDocSize(context.reader(), atomicReader.maxDoc()),
atomicReader);
-  }
-
-  private static AtomicReader setDocSize(AtomicReader reader, final int count) {
-    return new FilterAtomicReader(reader) {
-      @Override
-      public Bits getLiveDocs() {
-        return new Bits() {
-          @Override
-          public boolean get(int index) {
-            return true;
-          }
-
-          @Override
-          public int length() {
-            return count;
-          }
-        };
-      }
-
-      @Override
-      public int numDocs() {
-        return count;
-      }
-
-      @Override
-      public int maxDoc() {
-        return count;
-      }
-
-      @Override
-      public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-        // Do nothing
-      }
-    };
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
deleted file mode 100644
index 15700bc..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
+++ /dev/null
@@ -1,223 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.hadoop.util.Progressable;
-import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.BufferedIndexOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-
-/**
- * The {@link ProgressableDirectory} allows for progress to be recorded while
- * Lucene is blocked and merging. This prevents the Task from being killed after
- * not reporting progress because of the blocked merge.
- */
-public class ProgressableDirectory extends Directory {
-
-  private static final Log LOG = LogFactory.getLog(ProgressableDirectory.class);
-
-  private final Directory _directory;
-  private final Progressable _progressable;
-
-  public ProgressableDirectory(Directory directory, Progressable progressable) {
-    _directory = directory;
-    if (progressable == null) {
-      LOG.warn("Progressable is null.");
-      _progressable = new Progressable() {
-        @Override
-        public void progress() {
-
-        }
-      };
-    } else {
-      _progressable = progressable;
-    }
-  }
-
-  @Override
-  public void clearLock(String name) throws IOException {
-    _directory.clearLock(name);
-  }
-
-  @Override
-  public void close() throws IOException {
-    _directory.close();
-  }
-
-  private IndexInput wrapInput(String name, IndexInput openInput) {
-    return new ProgressableIndexInput(name, openInput, 16384, _progressable);
-  }
-
-  private IndexOutput wrapOutput(IndexOutput createOutput) {
-    return new ProgressableIndexOutput(createOutput, _progressable);
-  }
-
-  public IndexOutput createOutput(String name, IOContext context) throws IOException {
-    return wrapOutput(_directory.createOutput(name, context));
-  }
-
-  @Override
-  public void deleteFile(String name) throws IOException {
-    _directory.deleteFile(name);
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    return _directory.equals(obj);
-  }
-
-  @Override
-  public boolean fileExists(String name) throws IOException {
-    return _directory.fileExists(name);
-  }
-
-  @Override
-  public long fileLength(String name) throws IOException {
-    return _directory.fileLength(name);
-  }
-
-  @Override
-  public LockFactory getLockFactory() {
-    return _directory.getLockFactory();
-  }
-
-  @Override
-  public String getLockID() {
-    return _directory.getLockID();
-  }
-
-  @Override
-  public int hashCode() {
-    return _directory.hashCode();
-  }
-
-  @Override
-  public String[] listAll() throws IOException {
-    return _directory.listAll();
-  }
-
-  @Override
-  public Lock makeLock(String name) {
-    return _directory.makeLock(name);
-  }
-
-  @Override
-  public IndexInput openInput(String name, IOContext context) throws IOException {
-    return wrapInput(name, _directory.openInput(name, context));
-  }
-
-  @Override
-  public void setLockFactory(LockFactory lockFactory) throws IOException {
-    _directory.setLockFactory(lockFactory);
-  }
-
-  @Override
-  public void sync(Collection<String> names) throws IOException {
-    _directory.sync(names);
-  }
-
-  @Override
-  public String toString() {
-    return _directory.toString();
-  }
-
-  static class ProgressableIndexOutput extends BufferedIndexOutput {
-
-    private Progressable _progressable;
-    private IndexOutput _indexOutput;
-
-    public ProgressableIndexOutput(IndexOutput indexOutput, Progressable progressable) {
-      _indexOutput = indexOutput;
-      _progressable = progressable;
-    }
-
-    @Override
-    protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
-      _indexOutput.writeBytes(b, offset, len);
-      _progressable.progress();
-    }
-
-    @Override
-    public long length() throws IOException {
-      return _indexOutput.length();
-    }
-
-    @Override
-    public void close() throws IOException {
-      super.close();
-      _indexOutput.close();
-      _progressable.progress();
-    }
-
-  }
-
-  static class ProgressableIndexInput extends BufferedIndexInput {
-
-    private IndexInput _indexInput;
-    private final long _length;
-    private Progressable _progressable;
-
-    ProgressableIndexInput(String name, IndexInput indexInput, int buffer, Progressable progressable)
{
-      super("ProgressableIndexInput(" + indexInput.toString() + ")", buffer);
-      _indexInput = indexInput;
-      _length = indexInput.length();
-      _progressable = progressable;
-    }
-
-    @Override
-    protected void readInternal(byte[] b, int offset, int length) throws IOException {
-      long filePointer = getFilePointer();
-      if (filePointer != _indexInput.getFilePointer()) {
-        _indexInput.seek(filePointer);
-      }
-      _indexInput.readBytes(b, offset, length);
-      _progressable.progress();
-    }
-
-    @Override
-    protected void seekInternal(long pos) throws IOException {
-
-    }
-
-    @Override
-    public void close() throws IOException {
-      _indexInput.close();
-    }
-
-    @Override
-    public long length() {
-      return _length;
-    }
-
-    @Override
-    public ProgressableIndexInput clone() {
-      ProgressableIndexInput clone = (ProgressableIndexInput) super.clone();
-      clone._indexInput = (IndexInput) _indexInput.clone();
-      return clone;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/RateCounter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/RateCounter.java
b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/RateCounter.java
deleted file mode 100644
index 777cb14..0000000
--- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/RateCounter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.mapreduce.Counter;
-
-/**
- * This turns a standard hadoop counter into a rate counter.
- */
-public class RateCounter {
-
-  private final Counter _counter;
-  private final long _reportTime;
-  private long _lastReport;
-  private long _count = 0;
-
-  public RateCounter(Counter counter) {
-    _counter = counter;
-    _lastReport = System.nanoTime();
-    _reportTime = TimeUnit.SECONDS.toNanos(5);
-  }
-
-  public void mark() {
-    mark(1l);
-  }
-
-  public void mark(long n) {
-    long now = System.nanoTime();
-    if (_lastReport + _reportTime < now) {
-      long seconds = TimeUnit.NANOSECONDS.toSeconds(now - _lastReport);
-      long rate = _count / seconds;
-      _counter.setValue(rate);
-      _lastReport = System.nanoTime();
-      _count = 0;
-    }
-    _count += n;
-  }
-
-  public void close() {
-    _counter.setValue(0);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
deleted file mode 100644
index 89458f8..0000000
--- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.mapreduce.lib;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.blur.mapreduce.lib.CsvBlurDriver.ControllerPool;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.compress.SnappyCodec;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Test;
-
-public class CsvBlurDriverTest {
-
-  protected String tableUri = "file:///tmp/tmppath";
-  protected int shardCount = 13;
-
-  @Test
-  public void testCsvBlurDriverTestFail1() throws Exception {
-    Configuration configuration = new Configuration();
-    ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
-      @Override
-      public Iface getClient(String controllerConnectionStr) {
-        return null;
-      }
-    };
-    AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
-    assertNull(CsvBlurDriver.setupJob(configuration, controllerPool, ref, new String[] {}));
-  }
-
-  @Test
-  public void testCsvBlurDriverTest() throws Exception {
-    Configuration configurationSetup = new Configuration();
-    ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
-      @Override
-      public Iface getClient(String controllerConnectionStr) {
-        return getMockIface();
-      }
-    };
-    AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
-    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010",
"-d", "family1",
-        "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1",
"-i",
-        "file:///tmp/test2");
-    assertNotNull(job);
-    Configuration configuration = job.getConfiguration();
-    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
-    assertEquals(tableDescriptor.getName(), "table1");
-    Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
-    assertEquals(2, inputs.size());
-    Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
-    assertEquals(2, familyAndColumnNameMap.size());
-  }
-
-  @Test
-  public void testCsvBlurDriverTest2() throws Exception {
-    Configuration configurationSetup = new Configuration();
-    ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
-      @Override
-      public Iface getClient(String controllerConnectionStr) {
-        return getMockIface();
-      }
-    };
-    AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
-    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010",
"-d", "family1",
-        "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1",
"-i",
-        "file:///tmp/test2", "-S", "-C", "1000000", "2000000");
-    assertNotNull(job);
-    Configuration configuration = job.getConfiguration();
-    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
-    assertEquals(tableDescriptor.getName(), "table1");
-    Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
-    assertEquals(2, inputs.size());
-    Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
-    assertEquals(2, familyAndColumnNameMap.size());
-  }
-
-  @Test
-  public void testCsvBlurDriverTest3() throws Exception {
-    Configuration configurationSetup = new Configuration();
-    ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
-      @Override
-      public Iface getClient(String controllerConnectionStr) {
-        return getMockIface();
-      }
-    };
-    AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
-    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010",
"-d", "family1",
-        "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1",
"-i",
-        "file:///tmp/test2", "-S", "-C", "1000000", "2000000", "-p", "SNAPPY");
-    assertNotNull(job);
-    Configuration configuration = job.getConfiguration();
-    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
-    assertEquals(tableDescriptor.getName(), "table1");
-    Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
-    assertEquals(2, inputs.size());
-    Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
-    assertEquals(2, familyAndColumnNameMap.size());
-    assertEquals("true", configuration.get(CsvBlurDriver.MAPRED_COMPRESS_MAP_OUTPUT));
-    assertEquals(SnappyCodec.class.getName(), configuration.get(CsvBlurDriver.MAPRED_MAP_OUTPUT_COMPRESSION_CODEC));
-  }
-
-  protected Iface getMockIface() {
-    InvocationHandler handler = new InvocationHandler() {
-
-      @Override
-      public Object invoke(Object o, Method method, Object[] args) throws Throwable {
-        if (method.getName().equals("describe")) {
-          TableDescriptor tableDescriptor = new TableDescriptor();
-          tableDescriptor.setName((String) args[0]);
-          tableDescriptor.setTableUri(tableUri);
-          tableDescriptor.setShardCount(shardCount);
-          return tableDescriptor;
-        }
-        throw new RuntimeException("not implemented.");
-      }
-    };
-    return (Iface) Proxy.newProxyInstance(Iface.class.getClassLoader(), new Class[] { Iface.class
}, handler);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurMapperTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurMapperTest.java
b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurMapperTest.java
deleted file mode 100644
index 47aa8e5..0000000
--- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurMapperTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-
-import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CsvBlurMapperTest {
-
-  private MapDriver<Writable, Text, Text, BlurMutate> _mapDriver;
-  private CsvBlurMapper _mapper;
-
-  @Before
-  public void setUp() throws IOException {
-    _mapper = new CsvBlurMapper();
-    _mapDriver = MapDriver.newMapDriver(_mapper);
-  }
-
-  @Test
-  public void testMapperWithFamilyInData() {
-    Configuration configuration = _mapDriver.getConfiguration();
-    CsvBlurMapper.setColumns(configuration, "cf1:col1,col2|cf2:col1,col2,col3");
-    _mapDriver.withInput(new LongWritable(), new Text("rowid1,record1,cf1,value1,value2"));
-    _mapDriver.withOutput(new Text("rowid1"), new BlurMutate(MUTATE_TYPE.REPLACE, "rowid1",
"record1", "cf1")
-        .addColumn("col1", "value1").addColumn("col2", "value2"));
-    _mapDriver.runTest();
-  }
-
-  @Test
-  public void testMapperFamilyPerPath() {
-    Configuration configuration = _mapDriver.getConfiguration();
-    CsvBlurMapper.setColumns(configuration, "cf1:col1,col2|cf2:col1,col2,col3");
-    CsvBlurMapper.addFamilyPath(configuration, "cf1", new Path("/"));
-    _mapper.setFamilyFromPath("cf1");
-
-    _mapDriver.withInput(new LongWritable(), new Text("rowid1,record1,value1,value2"));
-    _mapDriver.withOutput(new Text("rowid1"), new BlurMutate(MUTATE_TYPE.REPLACE, "rowid1",
"record1", "cf1")
-        .addColumn("col1", "value1").addColumn("col2", "value2"));
-    _mapDriver.runTest();
-  }
-  
-  @Test
-  public void testMapperAutoGenerateRecordId() {
-    Configuration configuration = _mapDriver.getConfiguration();
-    CsvBlurMapper.setAutoGenerateRecordIdAsHashOfData(configuration, true);
-    CsvBlurMapper.setColumns(configuration, "cf1:col1,col2|cf2:col1,col2,col3");
-    CsvBlurMapper.addFamilyPath(configuration, "cf1", new Path("/"));
-    _mapper.setFamilyFromPath("cf1");
-
-    _mapDriver.withInput(new LongWritable(), new Text("rowid1,value1,value2"));
-    _mapDriver.withOutput(new Text("rowid1"), new BlurMutate(MUTATE_TYPE.REPLACE, "rowid1",
"-25nqln3n2vb4cayex9y9tpxx3", "cf1")
-        .addColumn("col1", "value1").addColumn("col2", "value2"));
-    _mapDriver.runTest();
-  }
-  
-  @Test
-  public void testMapperAutoGenerateRowId() {
-    Configuration configuration = _mapDriver.getConfiguration();
-    CsvBlurMapper.setAutoGenerateRowIdAsHashOfData(configuration, true);
-    CsvBlurMapper.setColumns(configuration, "cf1:col1,col2|cf2:col1,col2,col3");
-    CsvBlurMapper.addFamilyPath(configuration, "cf1", new Path("/"));
-    _mapper.setFamilyFromPath("cf1");
-
-    _mapDriver.withInput(new LongWritable(), new Text("record1,value1,value2"));
-    _mapDriver.withOutput(new Text("-50b4uzohynr7j7s9pve7ytz66"), new BlurMutate(MUTATE_TYPE.REPLACE,
"-50b4uzohynr7j7s9pve7ytz66", "record1", "cf1")
-        .addColumn("col1", "value1").addColumn("col2", "value2"));
-    _mapDriver.runTest();
-  }
-  
-  @Test
-  public void testMapperAutoGenerateRowIdAndRecordId() {
-    Configuration configuration = _mapDriver.getConfiguration();
-    CsvBlurMapper.setAutoGenerateRecordIdAsHashOfData(configuration, true);
-    CsvBlurMapper.setAutoGenerateRowIdAsHashOfData(configuration, true);
-    CsvBlurMapper.setColumns(configuration, "cf1:col1,col2|cf2:col1,col2,col3");
-    CsvBlurMapper.addFamilyPath(configuration, "cf1", new Path("/"));
-    _mapper.setFamilyFromPath("cf1");
-
-    _mapDriver.withInput(new LongWritable(), new Text("value1,value2"));
-    _mapDriver.withOutput(new Text("5q0tme15ph3h5pns8sv3u5wy2"), new BlurMutate(MUTATE_TYPE.REPLACE,
"5q0tme15ph3h5pns8sv3u5wy2", "5q0tme15ph3h5pns8sv3u5wy2", "cf1")
-        .addColumn("col1", "value1").addColumn("col2", "value2"));
-    _mapDriver.runTest();
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e8165b77/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index af3172f..0d43a28 100644
--- a/pom.xml
+++ b/pom.xml
@@ -508,6 +508,7 @@ under the License.
 				<module>blur-thrift</module>
 				<module>blur-query</module>
 				<module>blur-store</module>
+				<module>blur-mapred-common</module>
 				<module>blur-mapred-hadoop1</module>
 				<module>blur-util</module>
 				<module>blur-status</module>
@@ -534,6 +535,7 @@ under the License.
 				<module>blur-thrift</module>
 				<module>blur-query</module>
 				<module>blur-store</module>
+				<module>blur-mapred-common</module>
 				<module>blur-mapred-hadoop1</module>
 				<module>blur-util</module>
 				<module>blur-status</module>
@@ -560,6 +562,7 @@ under the License.
 				<module>blur-thrift</module>
 				<module>blur-query</module>
 				<module>blur-store</module>
+				<module>blur-mapred-common</module>
 				<module>blur-mapred-hadoop2</module>
 				<module>blur-util</module>
 				<module>blur-status</module>


Mime
View raw message