incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Making the counter more clear. Adding a column counter and lucene field counter. Removing the field counter.
Date Mon, 05 Aug 2013 11:16:16 GMT
Making the counter more clear.  Adding a column counter and lucene field counter.  Removing
the field counter.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/08dfc4b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/08dfc4b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/08dfc4b0

Branch: refs/heads/master
Commit: 08dfc4b0e8ec201cbc059487f57fd4a890f1abc1
Parents: 795a13a
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Aug 5 07:15:20 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Aug 5 07:15:20 2013 -0400

----------------------------------------------------------------------
 .../java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java     | 4 ++--
 .../main/java/org/apache/blur/mapreduce/lib/BlurCounters.java  | 2 +-
 .../java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java   | 6 +++++-
 .../main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java | 2 +-
 4 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
index a796810..037edec 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java
@@ -32,7 +32,7 @@ public abstract class BaseBlurMapper<KEY, VALUE> extends Mapper<KEY,
VALUE, Text
   protected BlurMutate _mutate;
   protected Text _key;
   protected Counter _recordCounter;
-  protected Counter _fieldCounter;
+  protected Counter _columnCounter;
 
   @Override
   protected void setup(Context context) throws IOException, InterruptedException {
@@ -40,7 +40,7 @@ public abstract class BaseBlurMapper<KEY, VALUE> extends Mapper<KEY,
VALUE, Text
     _mutate.setRecord(new BlurRecord());
     _key = new Text();
     _recordCounter = context.getCounter(BlurCounters.RECORD_COUNT);
-    _fieldCounter = context.getCounter(BlurCounters.FIELD_COUNT);
+    _columnCounter = context.getCounter(BlurCounters.COLUMN_COUNT);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
index 4d008e7..0691dce 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java
@@ -21,6 +21,6 @@ package org.apache.blur.mapreduce.lib;
  * The enum class used for all the internal counters during map reduce jobs.
  */
 public enum BlurCounters {
-  RECORD_COUNT, FIELD_COUNT, ROW_COUNT, RECORD_RATE, COPY_RATE, ROW_RATE, RECORD_DUPLICATE_COUNT,
ROW_OVERFLOW_COUNT, ROW_DELETE_COUNT
+  RECORD_COUNT, LUCENE_FIELD_COUNT, ROW_COUNT, RECORD_RATE, COPY_RATE, ROW_RATE, RECORD_DUPLICATE_COUNT,
ROW_OVERFLOW_COUNT, ROW_DELETE_COUNT, COLUMN_COUNT
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index eb8112b..39d670b 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -352,6 +352,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
     private final Path _newIndex;
     private final boolean _indexLocally;
     private final boolean _optimizeInFlight;
+    private Counter _columnCount = emptyCounter();
     private Counter _fieldCount = emptyCounter();
     private Counter _recordCount = emptyCounter();
     private Counter _rowCount = emptyCounter();
@@ -367,6 +368,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
     private File _localTmpPath;
     private ProgressableDirectory _localTmpDir;
     private String _deletedRowId;
+    
 
     public BlurRecordWriter(Configuration configuration, BlurAnalyzer blurAnalyzer, int attemptId,
String tmpDirName)
         throws IOException {
@@ -424,7 +426,8 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
 
     private void setupCounter() {
       GetCounter getCounter = BlurOutputFormat.getGetCounter();
-      _fieldCount = getCounter.getCounter(BlurCounters.FIELD_COUNT);
+      _fieldCount = getCounter.getCounter(BlurCounters.LUCENE_FIELD_COUNT);
+      _columnCount = getCounter.getCounter(BlurCounters.COLUMN_COUNT);
       _recordCount = getCounter.getCounter(BlurCounters.RECORD_COUNT);
       _recordDuplicateCount = getCounter.getCounter(BlurCounters.RECORD_DUPLICATE_COUNT);
       _rowCount = getCounter.getCounter(BlurCounters.ROW_COUNT);
@@ -443,6 +446,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
         _deletedRowId = blurRecord.getRowId();
         return;
       }
+      _columnCount.increment(record.getColumns().size());
       Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _analyzer);
       if (_documents.size() == 0) {
         document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
index a8158af..8de2e98 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
@@ -414,7 +414,7 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text>
{
 
     for (int i = 0; i < columnNames.size(); i++) {
       record.addColumn(columnNames.get(i), list.get(i + offset));
-      _fieldCounter.increment(1);
+      _columnCounter.increment(1);
     }
     _key.set(record.getRowId());
     _mutate.setMutateType(MUTATE_TYPE.REPLACE);


Mime
View raw message