incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/7] git commit: Removing old code, deprecating code that should not be used, and adding javadoc.
Date Thu, 16 May 2013 20:45:19 GMT
Removing old code, deprecating code that should not be used, and adding javadoc.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0cc4d72e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0cc4d72e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0cc4d72e

Branch: refs/heads/0.1.5
Commit: 0cc4d72eb68ce6feb224e75e73928aae23f5f719
Parents: 6a9fde0
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed May 15 16:22:45 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed May 15 16:22:45 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/mapred/BlurInputFormat.java    |   42 ---
 .../apache/blur/mapred/BlurOutputCommitter.java    |   29 --
 .../org/apache/blur/mapred/BlurRecordReader.java   |  108 ------
 .../java/org/apache/blur/mapreduce/BlurColumn.java |  109 ------
 .../java/org/apache/blur/mapreduce/BlurMapper.java |    3 +
 .../java/org/apache/blur/mapreduce/BlurMutate.java |  157 --------
 .../java/org/apache/blur/mapreduce/BlurRecord.java |  178 ----------
 .../org/apache/blur/mapreduce/BlurReducer.java     |    6 +-
 .../java/org/apache/blur/mapreduce/BlurTask.java   |    2 +
 .../java/org/apache/blur/mapreduce/IOUtil.java     |   58 ---
 .../blur/mapreduce/ProgressableDirectory.java      |  269 --------------
 .../mapreduce/csv/BlurExampleIndexerRebuild.java   |   79 ----
 .../mapreduce/csv/BlurExampleIndexerUpdate.java    |   62 ----
 .../apache/blur/mapreduce/csv/CsvBlurMapper.java   |  174 ---------
 .../apache/blur/mapreduce/lib/BaseBlurMapper.java  |   49 +++
 .../org/apache/blur/mapreduce/lib/BlurColumn.java  |  109 ++++++
 .../apache/blur/mapreduce/lib/BlurCounters.java    |   26 ++
 .../apache/blur/mapreduce/lib/BlurInputFormat.java |   71 ----
 .../apache/blur/mapreduce/lib/BlurInputSplit.java  |  157 --------
 .../org/apache/blur/mapreduce/lib/BlurMutate.java  |  157 ++++++++
 .../blur/mapreduce/lib/BlurOutputFormat.java       |   30 ++-
 .../org/apache/blur/mapreduce/lib/BlurRecord.java  |  178 ++++++++++
 .../blur/mapreduce/lib/BlurRecordReader.java       |    1 -
 .../apache/blur/mapreduce/lib/CsvBlurDriver.java   |   66 ++++
 .../apache/blur/mapreduce/lib/CsvBlurMapper.java   |  230 ++++++++++++
 .../blur/mapreduce/lib/DefaultBlurReducer.java     |    1 -
 .../java/org/apache/blur/mapreduce/lib/IOUtil.java |   58 +++
 .../blur/mapreduce/lib/ProgressableDirectory.java  |  275 +++++++++++++++
 .../java/org/apache/blur/mapreduce/lib/Utils.java  |   74 ----
 .../apache/blur/mapred/BlurInputFormatTest.java    |   82 -----
 .../org/apache/blur/mapreduce/BlurReducerTest.java |   57 ---
 .../org/apache/blur/mapreduce/BlurTaskTest.java    |    8 +-
 .../blur/mapreduce/lib/BlurInputFormatTest.java    |  145 --------
 .../blur/mapreduce/lib/BlurOutputFormatTest.java   |    2 -
 .../blur/mapreduce/lib/CsvBlurMapperTest.java      |   52 +++
 35 files changed, 1237 insertions(+), 1867 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurInputFormat.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurInputFormat.java b/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurInputFormat.java
deleted file mode 100644
index db17abf..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurInputFormat.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.apache.blur.mapred;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.blur.mapreduce.BlurRecord;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.InputFormat;
-
-
-public abstract class BlurInputFormat implements InputFormat<Text, BlurRecord> {
-
-//  @Override
-//  public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
-//    List<?> splits = new ArrayList<Object>();
-//    Path[] paths = FileInputFormat.getInputPaths(job);
-//    for (Path path : paths) {
-//      org.apache.blur.mapreduce.lib.BlurInputFormat.findAllSegments((Configuration) job, path, splits);
-//    }
-//    return splits.toArray(new InputSplit[] {});
-//  }
-//
-//  @Override
-//  public RecordReader<Text, BlurRecord> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
-//    reporter.setStatus(split.toString());
-//    return new BlurRecordReader(split, job);
-//  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurOutputCommitter.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurOutputCommitter.java b/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurOutputCommitter.java
deleted file mode 100644
index 1c3d2ca..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurOutputCommitter.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.apache.blur.mapred;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapred.TaskAttemptContext;
-
-public class BlurOutputCommitter extends AbstractOutputCommitter {
-
-  @Override
-  public void setupTask(TaskAttemptContext taskContext) throws IOException {
-
-  }
-
-  @Override
-  public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
-    return false;
-  }
-
-  @Override
-  public void commitTask(TaskAttemptContext taskContext) throws IOException {
-
-  }
-
-  @Override
-  public void abortTask(TaskAttemptContext taskContext) throws IOException {
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurRecordReader.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurRecordReader.java b/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurRecordReader.java
deleted file mode 100644
index 7b34289..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapred/BlurRecordReader.java
+++ /dev/null
@@ -1,108 +0,0 @@
-package org.apache.blur.mapred;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-
-import org.apache.blur.mapreduce.BlurRecord;
-import org.apache.blur.mapreduce.lib.BlurInputSplit;
-import org.apache.blur.mapreduce.lib.Utils;
-import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.utils.RowDocumentUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.store.Directory;
-
-
-public abstract class BlurRecordReader implements RecordReader<Text, BlurRecord> {
-
-//  private IndexReader reader;
-//  private Directory directory;
-//  private int startingDocId;
-//  private int endingDocId;
-//  private int position;
-//
-//  public BlurRecordReader(InputSplit split, JobConf job) throws IOException {
-//    BlurInputSplit blurSplit = (BlurInputSplit) split;
-//    Path path = blurSplit.getIndexPath();
-//    String segmentName = blurSplit.getSegmentName();
-//    startingDocId = blurSplit.getStartingDocId();
-//    endingDocId = blurSplit.getEndingDocId();
-//    directory = new HdfsDirectory(path);
-//
-//    IndexCommit commit = Utils.findLatest(directory);
-//    reader = Utils.openSegmentReader(directory, commit, segmentName, Utils.getTermInfosIndexDivisor(job));
-//    int maxDoc = reader.maxDoc();
-//    if (endingDocId >= maxDoc) {
-//      endingDocId = maxDoc - 1;
-//    }
-//    position = startingDocId - 1;
-//  }
-//
-//  @Override
-//  public boolean next(Text key, BlurRecord value) throws IOException {
-//    do {
-//      position++;
-//      if (position > endingDocId) {
-//        return false;
-//      }
-//    } while (reader.isDeleted(position));
-//    readDocument(key, value);
-//    return true;
-//  }
-//
-//  private void readDocument(Text rowid, BlurRecord record) throws CorruptIndexException, IOException {
-//    Document document = reader.document(position);
-//    record.reset();
-//    rowid.set(RowDocumentUtil.readRecord(document, record));
-//  }
-//
-//  @Override
-//  public Text createKey() {
-//    return new Text();
-//  }
-//
-//  @Override
-//  public BlurRecord createValue() {
-//    return new BlurRecord();
-//  }
-//
-//  @Override
-//  public long getPos() throws IOException {
-//    return position;
-//  }
-//
-//  @Override
-//  public void close() throws IOException {
-//    reader.close();
-//    directory.close();
-//  }
-//
-//  @Override
-//  public float getProgress() throws IOException {
-//    int total = endingDocId - startingDocId;
-//    return (float) position / (float) total;
-//  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurColumn.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurColumn.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurColumn.java
deleted file mode 100644
index e394ac0..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurColumn.java
+++ /dev/null
@@ -1,109 +0,0 @@
-package org.apache.blur.mapreduce;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-
-public class BlurColumn implements Writable {
-
-  private String name;
-  private String value;
-
-  public BlurColumn() {
-  }
-
-  public BlurColumn(String name, String value) {
-    this.name = name;
-    this.value = value;
-  }
-
-  public boolean hasNull() {
-    if (name == null || value == null) {
-      return true;
-    }
-    return false;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    name = IOUtil.readString(in);
-    value = IOUtil.readString(in);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    IOUtil.writeString(out, name);
-    IOUtil.writeString(out, value);
-  }
-
-  public String getValue() {
-    return value;
-  }
-
-  public void setValue(String value) {
-    this.value = value;
-  }
-
-  @Override
-  public String toString() {
-    return "{name=" + name + ", value=" + value + "}";
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + ((name == null) ? 0 : name.hashCode());
-    result = prime * result + ((value == null) ? 0 : value.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    BlurColumn other = (BlurColumn) obj;
-    if (name == null) {
-      if (other.name != null)
-        return false;
-    } else if (!name.equals(other.name))
-      return false;
-    if (value == null) {
-      if (other.value != null)
-        return false;
-    } else if (!value.equals(other.value))
-      return false;
-    return true;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMapper.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMapper.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMapper.java
index 9a3575e..2c4ec1f 100644
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMapper.java
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMapper.java
@@ -18,10 +18,13 @@ package org.apache.blur.mapreduce;
  */
 import java.io.IOException;
 
+import org.apache.blur.mapreduce.lib.BlurMutate;
+import org.apache.blur.mapreduce.lib.BlurRecord;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.Mapper;
 
+@Deprecated
 public abstract class BlurMapper<KEY, VALUE> extends Mapper<KEY, VALUE, Text, BlurMutate> {
 
   protected BlurMutate _mutate;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMutate.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMutate.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMutate.java
deleted file mode 100644
index c7615ea..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurMutate.java
+++ /dev/null
@@ -1,157 +0,0 @@
-package org.apache.blur.mapreduce;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-
-public class BlurMutate implements Writable {
-
-  public enum MUTATE_TYPE {
-    /*ADD(0), UPDATE(1),*/ DELETE(2), REPLACE(3);
-    private int _value;
-
-    private MUTATE_TYPE(int value) {
-      _value = value;
-    }
-
-    public int getValue() {
-      return _value;
-    }
-
-    public MUTATE_TYPE find(int value) {
-      switch (value) {
-   // @TODO Updates through MR is going to be disabled
-//      case 0:
-//        return ADD;
-//      case 1:
-//        return UPDATE;
-      case 2:
-        return DELETE;
-      case 3:
-        return REPLACE;
-      default:
-        throw new RuntimeException("Value [" + value + "] not found.");
-      }
-    }
-  }
-
-  private MUTATE_TYPE _mutateType = MUTATE_TYPE.REPLACE;
-  private BlurRecord _record = new BlurRecord();
-
-  public BlurMutate() {
-
-  }
-
-  public BlurMutate(MUTATE_TYPE type, BlurRecord record) {
-    _mutateType = type;
-    _record = record;
-  }
-
-  public BlurMutate(MUTATE_TYPE type, String rowId, String recordId) {
-    _mutateType = type;
-    _record.setRowId(rowId);
-    _record.setRecordId(recordId);
-  }
-
-  public BlurMutate(MUTATE_TYPE type, String rowId, String recordId, String family) {
-    _mutateType = type;
-    _record.setRowId(rowId);
-    _record.setRecordId(recordId);
-    _record.setFamily(family);
-  }
-
-  public BlurMutate addColumn(BlurColumn column) {
-    _record.addColumn(column);
-    return this;
-  }
-
-  public BlurMutate addColumn(String name, String value) {
-    return addColumn(new BlurColumn(name, value));
-  }
-
-  public BlurRecord getRecord() {
-    return _record;
-  }
-
-  public void setRecord(BlurRecord record) {
-    _record = record;
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    IOUtil.writeVInt(out, _mutateType.getValue());
-    _record.write(out);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    _mutateType.find(IOUtil.readVInt(in));
-    _record.readFields(in);
-  }
-
-  public MUTATE_TYPE getMutateType() {
-    return _mutateType;
-  }
-
-  public BlurMutate setMutateType(MUTATE_TYPE mutateType) {
-    _mutateType = mutateType;
-    return this;
-  }
-
-  @Override
-  public String toString() {
-    return "BlurMutate [mutateType=" + _mutateType + ", record=" + _record + "]";
-  }
-
-  public BlurMutate setFamily(String family) {
-    _record.setFamily(family);
-    return this;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + ((_mutateType == null) ? 0 : _mutateType.hashCode());
-    result = prime * result + ((_record == null) ? 0 : _record.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    BlurMutate other = (BlurMutate) obj;
-    if (_mutateType != other._mutateType)
-      return false;
-    if (_record == null) {
-      if (other._record != null)
-        return false;
-    } else if (!_record.equals(other._record))
-      return false;
-    return true;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurRecord.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurRecord.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurRecord.java
deleted file mode 100644
index 9d2868b..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurRecord.java
+++ /dev/null
@@ -1,178 +0,0 @@
-package org.apache.blur.mapreduce;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.blur.utils.ReaderBlurRecord;
-import org.apache.hadoop.io.Writable;
-
-public class BlurRecord implements Writable, ReaderBlurRecord {
-
-  private String _rowId;
-  private String _recordId;
-  private String _family;
-
-  private List<BlurColumn> _columns = new ArrayList<BlurColumn>();
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    _rowId = IOUtil.readString(in);
-    _recordId = IOUtil.readString(in);
-    _family = IOUtil.readString(in);
-    int size = IOUtil.readVInt(in);
-    _columns.clear();
-    for (int i = 0; i < size; i++) {
-      BlurColumn column = new BlurColumn();
-      column.readFields(in);
-      _columns.add(column);
-    }
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    IOUtil.writeString(out, _rowId);
-    IOUtil.writeString(out, _recordId);
-    IOUtil.writeString(out, _family);
-    IOUtil.writeVInt(out, _columns.size());
-    for (BlurColumn column : _columns) {
-      column.write(out);
-    }
-  }
-
-  public String getRowId() {
-    return _rowId;
-  }
-
-  public void setRowId(String rowId) {
-    this._rowId = rowId;
-  }
-
-  public String getRecordId() {
-    return _recordId;
-  }
-
-  public void setRecordId(String recordId) {
-    this._recordId = recordId;
-  }
-
-  public String getFamily() {
-    return _family;
-  }
-
-  public void setFamily(String family) {
-    this._family = family;
-  }
-
-  public List<BlurColumn> getColumns() {
-    return _columns;
-  }
-
-  public void setColumns(List<BlurColumn> columns) {
-    this._columns = columns;
-  }
-
-  public void clearColumns() {
-    _columns.clear();
-  }
-
-  public void addColumn(BlurColumn column) {
-    _columns.add(column);
-  }
-
-  public void addColumn(String name, String value) {
-    BlurColumn blurColumn = new BlurColumn();
-    blurColumn.setName(name);
-    blurColumn.setValue(value);
-    addColumn(blurColumn);
-  }
-
-  @Override
-  public void setRecordIdStr(String value) {
-    setRecordId(value);
-  }
-
-  @Override
-  public void setFamilyStr(String family) {
-    setFamily(family);
-  }
-
-  public void reset() {
-    clearColumns();
-    _rowId = null;
-    _recordId = null;
-    _family = null;
-  }
-
-  @Override
-  public void setRowIdStr(String rowId) {
-    setRowId(rowId);
-  }
-
-  @Override
-  public String toString() {
-    return "{rowId=" + _rowId + ", recordId=" + _recordId + ", family=" + _family + ", columns=" + _columns + "}";
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + ((_columns == null) ? 0 : _columns.hashCode());
-    result = prime * result + ((_family == null) ? 0 : _family.hashCode());
-    result = prime * result + ((_recordId == null) ? 0 : _recordId.hashCode());
-    result = prime * result + ((_rowId == null) ? 0 : _rowId.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    BlurRecord other = (BlurRecord) obj;
-    if (_columns == null) {
-      if (other._columns != null)
-        return false;
-    } else if (!_columns.equals(other._columns))
-      return false;
-    if (_family == null) {
-      if (other._family != null)
-        return false;
-    } else if (!_family.equals(other._family))
-      return false;
-    if (_recordId == null) {
-      if (other._recordId != null)
-        return false;
-    } else if (!_recordId.equals(other._recordId))
-      return false;
-    if (_rowId == null) {
-      if (other._rowId != null)
-        return false;
-    } else if (!_rowId.equals(other._rowId))
-      return false;
-    return true;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
index ba7a657..6871922 100644
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
@@ -37,10 +37,14 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;
 import org.apache.blur.manager.writer.TransactionRecorder;
-import org.apache.blur.mapreduce.BlurMutate.MUTATE_TYPE;
 import org.apache.blur.mapreduce.BlurTask.INDEXING_TYPE;
+import org.apache.blur.mapreduce.lib.BlurColumn;
+import org.apache.blur.mapreduce.lib.BlurMutate;
 import org.apache.blur.mapreduce.lib.BlurOutputFormat;
+import org.apache.blur.mapreduce.lib.BlurRecord;
 import org.apache.blur.mapreduce.lib.DefaultBlurReducer;
+import org.apache.blur.mapreduce.lib.ProgressableDirectory;
+import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE;
 import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.blur.thrift.generated.Column;
 import org.apache.blur.thrift.generated.TableDescriptor;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurTask.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurTask.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurTask.java
index 3af51ec..0d65a59 100644
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurTask.java
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurTask.java
@@ -29,6 +29,7 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus;
 import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
+import org.apache.blur.mapreduce.lib.BlurMutate;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
@@ -48,6 +49,7 @@ import org.apache.thrift.transport.TIOStreamTransport;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooKeeper;
 
+@Deprecated
 public class BlurTask implements Writable {
 
   public enum INDEXING_TYPE {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/IOUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/IOUtil.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/IOUtil.java
deleted file mode 100644
index 6acb009..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/IOUtil.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.blur.mapreduce;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-public class IOUtil {
-
-  public static final String UTF_8 = "UTF-8";
-
-  public static String readString(DataInput input) throws IOException {
-    int length = readVInt(input);
-    byte[] buffer = new byte[length];
-    input.readFully(buffer);
-    return new String(buffer, UTF_8);
-  }
-
-  public static void writeString(DataOutput output, String s) throws IOException {
-    byte[] bs = s.getBytes(UTF_8);
-    writeVInt(output, bs.length);
-    output.write(bs);
-  }
-
-  public static int readVInt(DataInput input) throws IOException {
-    byte b = input.readByte();
-    int i = b & 0x7F;
-    for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-      b = input.readByte();
-      i |= (b & 0x7F) << shift;
-    }
-    return i;
-  }
-
-  public static void writeVInt(DataOutput output, int i) throws IOException {
-    while ((i & ~0x7F) != 0) {
-      output.writeByte((byte) ((i & 0x7f) | 0x80));
-      i >>>= 7;
-    }
-    output.writeByte((byte) i);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/ProgressableDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/ProgressableDirectory.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/ProgressableDirectory.java
deleted file mode 100644
index 0930825..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/ProgressableDirectory.java
+++ /dev/null
@@ -1,269 +0,0 @@
-package org.apache.blur.mapreduce;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.hadoop.util.Progressable;
-import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-
-public class ProgressableDirectory extends Directory {
-
-  private Directory _directory;
-  private Progressable _progressable;
-
-  public ProgressableDirectory(Directory directory, Progressable progressable) {
-    _directory = directory;
-    _progressable = progressable;
-  }
-
-  @Override
-  public void clearLock(String name) throws IOException {
-    _directory.clearLock(name);
-  }
-
-  @Override
-  public void close() throws IOException {
-    _directory.close();
-  }
-
-  private IndexInput wrapInput(String name, IndexInput openInput) {
-    return new ProgressableIndexInput(name, openInput, 16384, _progressable);
-  }
-
-  private IndexOutput wrapOutput(IndexOutput createOutput) {
-    return new ProgressableIndexOutput(createOutput, _progressable);
-  }
-
-  public IndexOutput createOutput(String name, IOContext context) throws IOException {
-    return wrapOutput(_directory.createOutput(name, context));
-  }
-
-  @Override
-  public void deleteFile(String name) throws IOException {
-    _directory.deleteFile(name);
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    return _directory.equals(obj);
-  }
-
-  @Override
-  public boolean fileExists(String name) throws IOException {
-    return _directory.fileExists(name);
-  }
-
-  @Override
-  public long fileLength(String name) throws IOException {
-    return _directory.fileLength(name);
-  }
-
-  @Override
-  public LockFactory getLockFactory() {
-    return _directory.getLockFactory();
-  }
-
-  @Override
-  public String getLockID() {
-    return _directory.getLockID();
-  }
-
-  @Override
-  public int hashCode() {
-    return _directory.hashCode();
-  }
-
-  @Override
-  public String[] listAll() throws IOException {
-    return _directory.listAll();
-  }
-
-  @Override
-  public Lock makeLock(String name) {
-    return _directory.makeLock(name);
-  }
-
-  @Override
-  public IndexInput openInput(String name, IOContext context) throws IOException {
-    return wrapInput(name, _directory.openInput(name, context));
-  }
-
-  @Override
-  public void setLockFactory(LockFactory lockFactory) throws IOException {
-    _directory.setLockFactory(lockFactory);
-  }
-
-  @Override
-  public void sync(Collection<String> names) throws IOException {
-    _directory.sync(names);
-  }
-
-  @Override
-  public String toString() {
-    return _directory.toString();
-  }
-
-  static class ProgressableIndexOutput extends IndexOutput {
-
-    private Progressable _progressable;
-    private IndexOutput _indexOutput;
-
-    public ProgressableIndexOutput(IndexOutput indexOutput, Progressable progressable) {
-      _indexOutput = indexOutput;
-      _progressable = progressable;
-    }
-
-    @Override
-    public void close() throws IOException {
-      _indexOutput.close();
-      _progressable.progress();
-    }
-
-    @Override
-    public void copyBytes(DataInput input, long numBytes) throws IOException {
-      _indexOutput.copyBytes(input, numBytes);
-      _progressable.progress();
-    }
-
-    @Override
-    public void flush() throws IOException {
-      _indexOutput.flush();
-      _progressable.progress();
-    }
-
-    @Override
-    public long getFilePointer() {
-      return _indexOutput.getFilePointer();
-    }
-
-    @Override
-    public long length() throws IOException {
-      return _indexOutput.length();
-    }
-
-    @Override
-    public void seek(long pos) throws IOException {
-      _indexOutput.seek(pos);
-      _progressable.progress();
-    }
-
-    @Override
-    public void setLength(long length) throws IOException {
-      _indexOutput.setLength(length);
-      _progressable.progress();
-    }
-
-    @Override
-    public String toString() {
-      return _indexOutput.toString();
-    }
-
-    @Override
-    public void writeByte(byte b) throws IOException {
-      _indexOutput.writeByte(b);
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int offset, int length) throws IOException {
-      _indexOutput.writeBytes(b, offset, length);
-      _progressable.progress();
-    }
-
-    @Override
-    public void writeBytes(byte[] b, int length) throws IOException {
-      _indexOutput.writeBytes(b, length);
-      _progressable.progress();
-    }
-
-    @Override
-    public void writeInt(int i) throws IOException {
-      _indexOutput.writeInt(i);
-    }
-
-    @Override
-    public void writeLong(long i) throws IOException {
-      _indexOutput.writeLong(i);
-    }
-
-    @Override
-    public void writeString(String s) throws IOException {
-      _indexOutput.writeString(s);
-    }
-
-    @Override
-    public void writeStringStringMap(Map<String, String> map) throws IOException {
-      _indexOutput.writeStringStringMap(map);
-    }
-
-  }
-
-  static class ProgressableIndexInput extends BufferedIndexInput {
-
-    private IndexInput _indexInput;
-    private final long _length;
-    private Progressable _progressable;
-
-    ProgressableIndexInput(String name, IndexInput indexInput, int buffer, Progressable progressable) {
-      super(name, buffer);
-      _indexInput = indexInput;
-      _length = indexInput.length();
-      _progressable = progressable;
-    }
-
-    @Override
-    protected void readInternal(byte[] b, int offset, int length) throws IOException {
-      long filePointer = getFilePointer();
-      if (filePointer != _indexInput.getFilePointer()) {
-        _indexInput.seek(filePointer);
-      }
-      _indexInput.readBytes(b, offset, length);
-      _progressable.progress();
-    }
-
-    @Override
-    protected void seekInternal(long pos) throws IOException {
-
-    }
-
-    @Override
-    public void close() throws IOException {
-      _indexInput.close();
-    }
-
-    @Override
-    public long length() {
-      return _length;
-    }
-
-    @Override
-    public ProgressableIndexInput clone() {
-      ProgressableIndexInput clone = (ProgressableIndexInput) super.clone();
-      clone._indexInput = (IndexInput) _indexInput.clone();
-      return clone;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerRebuild.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerRebuild.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerRebuild.java
deleted file mode 100644
index 65ae8c8..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerRebuild.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.apache.blur.mapreduce.csv;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-
-import org.apache.blur.mapreduce.BlurTask;
-import org.apache.blur.mapreduce.BlurTask.INDEXING_TYPE;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.compress.DefaultCodec;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
-
-public class BlurExampleIndexerRebuild {
-
-  public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
-    Configuration configuration = new Configuration();
-    String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();
-    if (otherArgs.length != 2) {
-      System.err.println("Usage: blurindexer <in> <out>");
-      System.exit(2);
-    }
-
-    AnalyzerDefinition ad = new AnalyzerDefinition();
-    ad.defaultDefinition = new ColumnDefinition(StandardAnalyzer.class.getName(), true, null);
-
-    TableDescriptor descriptor = new TableDescriptor();
-    descriptor.analyzerDefinition = ad;
-    descriptor.compressionBlockSize = 32768;
-    descriptor.compressionClass = DefaultCodec.class.getName();
-    descriptor.isEnabled = true;
-    descriptor.name = "test-table";
-    descriptor.shardCount = 1;
-    descriptor.cluster = "default";
-    descriptor.tableUri = "./blur-testing";
-
-    BlurTask blurTask = new BlurTask();
-    blurTask.setTableDescriptor(descriptor);
-    blurTask.setIndexingType(INDEXING_TYPE.REBUILD);
-    blurTask.setOptimize(false);
-    Job job = blurTask.configureJob(configuration);
-    job.setJarByClass(BlurExampleIndexerRebuild.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-    job.setOutputFormatClass(TextOutputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
-    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1], "job-" + System.currentTimeMillis()));
-    long s = System.currentTimeMillis();
-    boolean waitForCompletion = job.waitForCompletion(true);
-    long e = System.currentTimeMillis();
-    System.out.println("Completed in [" + (e - s) + " ms]");
-    System.exit(waitForCompletion ? 0 : 1);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerUpdate.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerUpdate.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerUpdate.java
deleted file mode 100644
index 68bc076..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/BlurExampleIndexerUpdate.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.blur.mapreduce.csv;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-
-import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus;
-import org.apache.blur.mapreduce.BlurTask;
-import org.apache.blur.mapreduce.BlurTask.INDEXING_TYPE;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-
-
-public class BlurExampleIndexerUpdate {
-
-  public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
-    Configuration configuration = new Configuration();
-    String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();
-    if (otherArgs.length != 2) {
-      System.err.println("Usage: blurindexer <in> <out>");
-      System.exit(2);
-    }
-
-    ZookeeperClusterStatus status = new ZookeeperClusterStatus("localhost");
-    TableDescriptor descriptor = status.getTableDescriptor(false, "default", "test-table");
-
-    BlurTask blurTask = new BlurTask();
-    blurTask.setTableDescriptor(descriptor);
-    blurTask.setIndexingType(INDEXING_TYPE.UPDATE);
-    Job job = blurTask.configureJob(configuration);
-    job.setJarByClass(BlurExampleIndexerUpdate.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-    job.setOutputFormatClass(TextOutputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
-    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1], "job-" + System.currentTimeMillis()));
-    boolean waitForCompletion = job.waitForCompletion(true);
-    System.exit(waitForCompletion ? 0 : 1);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/CsvBlurMapper.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/CsvBlurMapper.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/CsvBlurMapper.java
deleted file mode 100644
index f41c40c..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/csv/CsvBlurMapper.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.blur.mapreduce.csv;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-
-import org.apache.blur.mapreduce.BlurMapper;
-import org.apache.blur.mapreduce.BlurMutate.MUTATE_TYPE;
-import org.apache.blur.mapreduce.BlurRecord;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-
-import com.google.common.base.Splitter;
-
-/**
- * 
- */
-public class CsvBlurMapper extends BlurMapper<LongWritable, Text> {
-
-  public static final String BLUR_CSV_SEPARATOR = "blur.csv.separator";
-  public static final String BLUR_CSV_FAMILY_COLUMN_PREFIX = "blur.csv.family.";
-  public static final String BLUR_CSV_FAMILIES = "blur.csv.families";
-
-  private Map<String, List<String>> columnNameMap;
-  private String separator = ",";
-  private Splitter splitter;
-
-  /**
-   * Adds the column layout for the given family.
-   * 
-   * @param job
-   *          the job to apply the layout.
-   * @param family
-   *          the family name.
-   * @param columns
-   *          the column names.
-   */
-  public static void addColumns(Job job, String family, String... columns) {
-    addColumns(job.getConfiguration(), family, columns);
-  }
-
-  /**
-   * Adds the column layout for the given family.
-   * 
-   * @param configuration
-   *          the configuration to apply the layout.
-   * @param family
-   *          the family name.
-   * @param columns
-   *          the column names.
-   */
-  public static void addColumns(Configuration configuration, String family, String... columns) {
-    Collection<String> families = new TreeSet<String>(configuration.getStringCollection(BLUR_CSV_FAMILIES));
-    families.add(family);
-    configuration.setStrings(BLUR_CSV_FAMILIES, families.toArray(new String[] {}));
-    configuration.setStrings(BLUR_CSV_FAMILY_COLUMN_PREFIX + family, columns);
-  }
-
-  /**
-   * Sets the separator of the file, by default it is ",".
-   * 
-   * @param job
-   *          the job to apply the separator change.
-   * @param separator
-   *          the separator.
-   */
-  public static void setSeparator(Job job, String separator) {
-    setSeparator(job.getConfiguration(), separator);
-  }
-
-  /**
-   * Sets the separator of the file, by default it is ",".
-   * 
-   * @param configuration
-   *          the configuration to apply the separator change.
-   * @param separator
-   *          the separator.
-   */
-  public static void setSeparator(Configuration configuration, String separator) {
-    configuration.set(BLUR_CSV_SEPARATOR, separator);
-  }
-
-  @Override
-  protected void setup(Context context) throws IOException, InterruptedException {
-    super.setup(context);
-    Configuration configuration = context.getConfiguration();
-    Collection<String> familyNames = configuration.getStringCollection(BLUR_CSV_FAMILIES);
-    columnNameMap = new HashMap<String, List<String>>();
-    for (String family : familyNames) {
-      String[] columnsNames = configuration.getStrings(BLUR_CSV_FAMILY_COLUMN_PREFIX + family);
-      columnNameMap.put(family, Arrays.asList(columnsNames));
-    }
-    splitter = Splitter.on(separator);
-    separator = configuration.get(BLUR_CSV_SEPARATOR, separator);
-  }
-
-  @Override
-  protected void map(LongWritable k, Text value, Context context) throws IOException, InterruptedException {
-    BlurRecord record = _mutate.getRecord();
-    record.clearColumns();
-    String str = value.toString();
-
-    Iterable<String> split = splitter.split(str);
-    List<String> list = toList(split);
-
-    if (list.size() < 3) {
-      throw new IOException("Record [" + str + "] too short.");
-    }
-
-    record.setRowId(list.get(0));
-    record.setRecordId(list.get(1));
-    String family = list.get(2);
-    record.setFamily(family);
-
-    List<String> columnNames = columnNameMap.get(family);
-    if (columnNames == null) {
-      throw new IOException("Family [" + family + "] is missing in the definition.");
-    }
-    if (list.size() - 3 != columnNames.size()) {
-      throw new IOException("Record [" + str + "] too short, does not match defined record [rowid,recordid,family"
-          + getColumnNames(columnNames) + "].");
-    }
-
-    for (int i = 0; i < columnNames.size(); i++) {
-      record.addColumn(columnNames.get(i), list.get(i + 3));
-      _fieldCounter.increment(1);
-    }
-    _key.set(record.getRowId());
-    _mutate.setMutateType(MUTATE_TYPE.REPLACE);
-    context.write(_key, _mutate);
-    _recordCounter.increment(1);
-    context.progress();
-  }
-
-  private String getColumnNames(List<String> columnNames) {
-    StringBuilder builder = new StringBuilder();
-    for (String c : columnNames) {
-      builder.append(',').append(c);
-    }
-    return builder.toString();
-  }
-
-  private List<String> toList(Iterable<String> split) {
-    List<String> lst = new ArrayList<String>();
-    for (String s : split) {
-      lst.add(s);
-    }
-    return lst;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
new file mode 100644
index 0000000..a796810
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
@@ -0,0 +1,49 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Mapper;
+
+/**
+ * Base mapper class for Blur map reduce classes.
+ * 
+ * @param <KEY>
+ * @param <VALUE>
+ */
+public abstract class BaseBlurMapper<KEY, VALUE> extends Mapper<KEY, VALUE, Text, BlurMutate> {
+  protected BlurMutate _mutate;
+  protected Text _key;
+  protected Counter _recordCounter;
+  protected Counter _fieldCounter;
+
+  @Override
+  protected void setup(Context context) throws IOException, InterruptedException {
+    _mutate = new BlurMutate();
+    _mutate.setRecord(new BlurRecord());
+    _key = new Text();
+    _recordCounter = context.getCounter(BlurCounters.RECORD_COUNT);
+    _fieldCounter = context.getCounter(BlurCounters.FIELD_COUNT);
+  }
+
+  @Override
+  protected abstract void map(KEY key, VALUE value, Context context) throws IOException, InterruptedException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurColumn.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurColumn.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurColumn.java
new file mode 100644
index 0000000..d32a3bd
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurColumn.java
@@ -0,0 +1,109 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+public class BlurColumn implements Writable {
+
+  private String name;
+  private String value;
+
+  public BlurColumn() {
+  }
+
+  public BlurColumn(String name, String value) {
+    this.name = name;
+    this.value = value;
+  }
+
+  public boolean hasNull() {
+    if (name == null || value == null) {
+      return true;
+    }
+    return false;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    name = IOUtil.readString(in);
+    value = IOUtil.readString(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    IOUtil.writeString(out, name);
+    IOUtil.writeString(out, value);
+  }
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+
+  @Override
+  public String toString() {
+    return "{name=" + name + ", value=" + value + "}";
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((name == null) ? 0 : name.hashCode());
+    result = prime * result + ((value == null) ? 0 : value.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlurColumn other = (BlurColumn) obj;
+    if (name == null) {
+      if (other.name != null)
+        return false;
+    } else if (!name.equals(other.name))
+      return false;
+    if (value == null) {
+      if (other.value != null)
+        return false;
+    } else if (!value.equals(other.value))
+      return false;
+    return true;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
new file mode 100644
index 0000000..6bd3407
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
@@ -0,0 +1,26 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * The enum class used for all the internal counters during map reduce jobs.
+ */
+public enum BlurCounters {
+  RECORD_COUNT, FIELD_COUNT
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputFormat.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputFormat.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputFormat.java
deleted file mode 100644
index b3419dd..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputFormat.java
+++ /dev/null
@@ -1,71 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.blur.mapreduce.BlurRecord;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputFormat;
-
-
-public abstract class BlurInputFormat extends InputFormat<Text, BlurRecord> {
-
-//  @SuppressWarnings("unchecked")
-//  @Override
-//  public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
-//    List<?> splits = new ArrayList<Object>();
-//    Path[] paths = FileInputFormat.getInputPaths(context);
-//    for (Path path : paths) {
-//      findAllSegments(context.getConfiguration(), path, splits);
-//    }
-//    return (List<InputSplit>) splits;
-//  }
-//
-//  @Override
-//  public RecordReader<Text, BlurRecord> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
-//    BlurRecordReader blurRecordReader = new BlurRecordReader();
-//    blurRecordReader.initialize(split, context);
-//    return blurRecordReader;
-//  }
-//
-//  public static void findAllSegments(Configuration configuration, Path path, List<?> splits) throws IOException {
-//    FileSystem fileSystem = path.getFileSystem(configuration);
-//    if (fileSystem.isFile(path)) {
-//      return;
-//    } else {
-//      FileStatus[] listStatus = fileSystem.listStatus(path);
-//      for (FileStatus status : listStatus) {
-//        Path p = status.getPath();
-//        HdfsDirectory directory = new HdfsDirectory(p);
-//        if (IndexReader.indexExists(directory)) {
-//          addSplits(directory, splits);
-//        } else {
-//          findAllSegments(configuration, p, splits);
-//        }
-//      }
-//    }
-//  }
-//
-//  @SuppressWarnings("unchecked")
-//  public static void addSplits(HdfsDirectory directory, @SuppressWarnings("rawtypes") List splits) throws IOException {
-//    IndexCommit commit = Utils.findLatest(directory);
-//    List<String> segments = Utils.getSegments(directory, commit);
-//    for (String segment : segments) {
-//      BlurInputSplit split = new BlurInputSplit(directory.getHdfsDirPath(), segment, 0, Integer.MAX_VALUE);
-//      splits.add(split);
-//    }
-//  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputSplit.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputSplit.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputSplit.java
deleted file mode 100644
index 7407ba4..0000000
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurInputSplit.java
+++ /dev/null
@@ -1,157 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputSplit;
-
-public class BlurInputSplit extends InputSplit implements Writable, org.apache.hadoop.mapred.InputSplit {
-
-  private int _endingDocId;
-  private int _startingDocId;
-  private String _segmentName;
-  private Path _path;
-
-  public BlurInputSplit() {
-
-  }
-
-  public BlurInputSplit(Path path, String segmentName, int startingDocId, int endingDocId) {
-    _endingDocId = endingDocId;
-    _startingDocId = startingDocId;
-    _segmentName = segmentName;
-    _path = path;
-  }
-
-  @Override
-  public long getLength() {
-    return _endingDocId - _startingDocId;
-  }
-
-  @Override
-  public String[] getLocations() {
-    return new String[] {};
-  }
-
-  public Path getIndexPath() {
-    return _path;
-  }
-
-  public String getSegmentName() {
-    return _segmentName;
-  }
-
-  public int getStartingDocId() {
-    return _startingDocId;
-  }
-
-  public int getEndingDocId() {
-    return _endingDocId;
-  }
-
-  public void setEndingDocId(int endingDocId) {
-    _endingDocId = endingDocId;
-  }
-
-  public void setStartingDocId(int startingDocId) {
-    _startingDocId = startingDocId;
-  }
-
-  public void setSegmentName(String segmentName) {
-    _segmentName = segmentName;
-  }
-
-  public void setPath(Path path) {
-    _path = path;
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(_startingDocId);
-    out.writeInt(_endingDocId);
-    writeString(out, _segmentName);
-    writeString(out, _path.toUri().toString());
-  }
-
-  private void writeString(DataOutput out, String s) throws IOException {
-    byte[] bs = s.getBytes();
-    out.writeInt(bs.length);
-    out.write(bs);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    _startingDocId = in.readInt();
-    _endingDocId = in.readInt();
-    _segmentName = readString(in);
-    _path = new Path(readString(in));
-  }
-
-  private String readString(DataInput in) throws IOException {
-    int length = in.readInt();
-    byte[] buf = new byte[length];
-    in.readFully(buf);
-    return new String(buf);
-  }
-
-  @Override
-  public String toString() {
-    return "path=" + _path + ", segmentName=" + _segmentName + ", startingDocId=" + _startingDocId + ", endingDocId=" + _endingDocId;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + _endingDocId;
-    result = prime * result + ((_path == null) ? 0 : _path.hashCode());
-    result = prime * result + ((_segmentName == null) ? 0 : _segmentName.hashCode());
-    result = prime * result + _startingDocId;
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    BlurInputSplit other = (BlurInputSplit) obj;
-    if (_endingDocId != other._endingDocId)
-      return false;
-    if (_path == null) {
-      if (other._path != null)
-        return false;
-    } else if (!_path.equals(other._path))
-      return false;
-    if (_segmentName == null) {
-      if (other._segmentName != null)
-        return false;
-    } else if (!_segmentName.equals(other._segmentName))
-      return false;
-    if (_startingDocId != other._startingDocId)
-      return false;
-    return true;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurMutate.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurMutate.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurMutate.java
new file mode 100644
index 0000000..3fdb97c
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurMutate.java
@@ -0,0 +1,157 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+public class BlurMutate implements Writable {
+
+  public enum MUTATE_TYPE {
+    /*ADD(0), UPDATE(1),*/ DELETE(2), REPLACE(3);
+    private int _value;
+
+    private MUTATE_TYPE(int value) {
+      _value = value;
+    }
+
+    public int getValue() {
+      return _value;
+    }
+
+    public MUTATE_TYPE find(int value) {
+      switch (value) {
+   // @TODO Updates through MR is going to be disabled
+//      case 0:
+//        return ADD;
+//      case 1:
+//        return UPDATE;
+      case 2:
+        return DELETE;
+      case 3:
+        return REPLACE;
+      default:
+        throw new RuntimeException("Value [" + value + "] not found.");
+      }
+    }
+  }
+
+  private MUTATE_TYPE _mutateType = MUTATE_TYPE.REPLACE;
+  private BlurRecord _record = new BlurRecord();
+
+  public BlurMutate() {
+
+  }
+
+  public BlurMutate(MUTATE_TYPE type, BlurRecord record) {
+    _mutateType = type;
+    _record = record;
+  }
+
+  public BlurMutate(MUTATE_TYPE type, String rowId, String recordId) {
+    _mutateType = type;
+    _record.setRowId(rowId);
+    _record.setRecordId(recordId);
+  }
+
+  public BlurMutate(MUTATE_TYPE type, String rowId, String recordId, String family) {
+    _mutateType = type;
+    _record.setRowId(rowId);
+    _record.setRecordId(recordId);
+    _record.setFamily(family);
+  }
+
+  public BlurMutate addColumn(BlurColumn column) {
+    _record.addColumn(column);
+    return this;
+  }
+
+  public BlurMutate addColumn(String name, String value) {
+    return addColumn(new BlurColumn(name, value));
+  }
+
+  public BlurRecord getRecord() {
+    return _record;
+  }
+
+  public void setRecord(BlurRecord record) {
+    _record = record;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    IOUtil.writeVInt(out, _mutateType.getValue());
+    _record.write(out);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    _mutateType.find(IOUtil.readVInt(in));
+    _record.readFields(in);
+  }
+
+  public MUTATE_TYPE getMutateType() {
+    return _mutateType;
+  }
+
+  public BlurMutate setMutateType(MUTATE_TYPE mutateType) {
+    _mutateType = mutateType;
+    return this;
+  }
+
+  @Override
+  public String toString() {
+    return "BlurMutate [mutateType=" + _mutateType + ", record=" + _record + "]";
+  }
+
+  public BlurMutate setFamily(String family) {
+    _record.setFamily(family);
+    return this;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((_mutateType == null) ? 0 : _mutateType.hashCode());
+    result = prime * result + ((_record == null) ? 0 : _record.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlurMutate other = (BlurMutate) obj;
+    if (_mutateType != other._mutateType)
+      return false;
+    if (_record == null) {
+      if (other._record != null)
+        return false;
+    } else if (!_record.equals(other._record))
+      return false;
+    return true;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index 9557fa2..435d072 100644
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -27,9 +27,6 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.LuceneVersionConstant;
 import org.apache.blur.manager.writer.TransactionRecorder;
-import org.apache.blur.mapreduce.BlurColumn;
-import org.apache.blur.mapreduce.BlurMutate;
-import org.apache.blur.mapreduce.BlurRecord;
 import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.blur.thrift.generated.Column;
 import org.apache.blur.thrift.generated.Record;
@@ -182,7 +179,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
     return new Path(configuration.get(BLUR_OUTPUT_PATH));
   }
 
-  
   static class BlurRecordWriter extends RecordWriter<Text, BlurMutate> {
 
     private static Log LOG = LogFactory.getLog(BlurRecordWriter.class);
@@ -193,12 +189,13 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
     private final BlurAnalyzer _analyzer;
     private final StringBuilder _builder = new StringBuilder();
 
-    public BlurRecordWriter(Configuration configuration, BlurAnalyzer blurAnalyzer, int shardId, String tmpDirName) throws IOException {
+    public BlurRecordWriter(Configuration configuration, BlurAnalyzer blurAnalyzer, int shardId, String tmpDirName)
+        throws IOException {
       Path tableOutput = BlurOutputFormat.getOutputPath(configuration);
       String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId);
       Path indexPath = new Path(tableOutput, shardName);
-      Path newIndex = new Path(indexPath,tmpDirName);
-      
+      Path newIndex = new Path(indexPath, tmpDirName);
+
       TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
       _analyzer = new BlurAnalyzer(tableDescriptor.getAnalyzerDefinition());
       IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, _analyzer);
@@ -251,4 +248,23 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
       _writer.close();
     }
   }
+
+  /**
+   * Sets up the output postion of the map reduce job. This does effect the map
+   * side of the job.
+   * 
+   * @param job
+   *          the job to setup.
+   * @param tableDescriptor
+   *          the table descriptor to write the output of the indexing job.
+   * @throws IOException
+   */
+  public static void setupJob(Job job, TableDescriptor tableDescriptor) throws IOException {
+    job.setReducerClass(DefaultBlurReducer.class);
+    job.setNumReduceTasks(tableDescriptor.getShardCount());
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(BlurMutate.class);
+    job.setOutputFormatClass(BlurOutputFormat.class);
+    setTableDescriptor(job, tableDescriptor);
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecord.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecord.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecord.java
new file mode 100644
index 0000000..7c12a76
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecord.java
@@ -0,0 +1,178 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.blur.utils.ReaderBlurRecord;
+import org.apache.hadoop.io.Writable;
+
+public class BlurRecord implements Writable, ReaderBlurRecord {
+
+  private String _rowId;
+  private String _recordId;
+  private String _family;
+
+  private List<BlurColumn> _columns = new ArrayList<BlurColumn>();
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    _rowId = IOUtil.readString(in);
+    _recordId = IOUtil.readString(in);
+    _family = IOUtil.readString(in);
+    int size = IOUtil.readVInt(in);
+    _columns.clear();
+    for (int i = 0; i < size; i++) {
+      BlurColumn column = new BlurColumn();
+      column.readFields(in);
+      _columns.add(column);
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    IOUtil.writeString(out, _rowId);
+    IOUtil.writeString(out, _recordId);
+    IOUtil.writeString(out, _family);
+    IOUtil.writeVInt(out, _columns.size());
+    for (BlurColumn column : _columns) {
+      column.write(out);
+    }
+  }
+
+  public String getRowId() {
+    return _rowId;
+  }
+
+  public void setRowId(String rowId) {
+    this._rowId = rowId;
+  }
+
+  public String getRecordId() {
+    return _recordId;
+  }
+
+  public void setRecordId(String recordId) {
+    this._recordId = recordId;
+  }
+
+  public String getFamily() {
+    return _family;
+  }
+
+  public void setFamily(String family) {
+    this._family = family;
+  }
+
+  public List<BlurColumn> getColumns() {
+    return _columns;
+  }
+
+  public void setColumns(List<BlurColumn> columns) {
+    this._columns = columns;
+  }
+
+  public void clearColumns() {
+    _columns.clear();
+  }
+
+  public void addColumn(BlurColumn column) {
+    _columns.add(column);
+  }
+
+  public void addColumn(String name, String value) {
+    BlurColumn blurColumn = new BlurColumn();
+    blurColumn.setName(name);
+    blurColumn.setValue(value);
+    addColumn(blurColumn);
+  }
+
+  @Override
+  public void setRecordIdStr(String value) {
+    setRecordId(value);
+  }
+
+  @Override
+  public void setFamilyStr(String family) {
+    setFamily(family);
+  }
+
+  public void reset() {
+    clearColumns();
+    _rowId = null;
+    _recordId = null;
+    _family = null;
+  }
+
+  @Override
+  public void setRowIdStr(String rowId) {
+    setRowId(rowId);
+  }
+
+  @Override
+  public String toString() {
+    return "{rowId=" + _rowId + ", recordId=" + _recordId + ", family=" + _family + ", columns=" + _columns + "}";
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((_columns == null) ? 0 : _columns.hashCode());
+    result = prime * result + ((_family == null) ? 0 : _family.hashCode());
+    result = prime * result + ((_recordId == null) ? 0 : _recordId.hashCode());
+    result = prime * result + ((_rowId == null) ? 0 : _rowId.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlurRecord other = (BlurRecord) obj;
+    if (_columns == null) {
+      if (other._columns != null)
+        return false;
+    } else if (!_columns.equals(other._columns))
+      return false;
+    if (_family == null) {
+      if (other._family != null)
+        return false;
+    } else if (!_family.equals(other._family))
+      return false;
+    if (_recordId == null) {
+      if (other._recordId != null)
+        return false;
+    } else if (!_recordId.equals(other._recordId))
+      return false;
+    if (_rowId == null) {
+      if (other._rowId != null)
+        return false;
+    } else if (!_rowId.equals(other._rowId))
+      return false;
+    return true;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecordReader.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecordReader.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecordReader.java
index e34a2b3..5f4fec6 100644
--- a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecordReader.java
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurRecordReader.java
@@ -16,7 +16,6 @@ package org.apache.blur.mapreduce.lib;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.blur.mapreduce.BlurRecord;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.RecordReader;
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0cc4d72e/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
new file mode 100644
index 0000000..c6dcf42
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
@@ -0,0 +1,66 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.thrift.TException;
+
+
+public class CsvBlurDriver {
+
+  public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException,
+      BlurException, TException {
+    Configuration configuration = new Configuration();
+    String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();
+    if (otherArgs.length != 4) {
+      System.err
+          .println("Usage: csvindexer <thrift controller connection str> <tablename> <in> <column family definitions>");
+      System.exit(2);
+    }
+
+    final String controllerConnectionStr = otherArgs[0];
+    final String tableName = otherArgs[1];
+    final String input = otherArgs[2];
+    final String columnDefs = otherArgs[3];
+
+    final Iface client = BlurClient.getClient(controllerConnectionStr);
+    TableDescriptor tableDescriptor = client.describe(tableName);
+
+    Job job = new Job(configuration, "Blur indexer [" + tableName + "] [" + input + "]");
+    job.setJarByClass(CsvBlurDriver.class);
+    job.setMapperClass(CsvBlurMapper.class);
+    job.setInputFormatClass(TextInputFormat.class);
+
+    FileInputFormat.addInputPath(job, new Path(input));
+    CsvBlurMapper.setColumns(job, columnDefs);
+    BlurOutputFormat.setupJob(job, tableDescriptor);
+
+    boolean waitForCompletion = job.waitForCompletion(true);
+    System.exit(waitForCompletion ? 0 : 1);
+  }
+}


Mime
View raw message