incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Changes needed to compile for cdh4-mr1
Date Tue, 15 Oct 2013 20:03:31 GMT
Updated Branches:
  refs/heads/master a8b67b34e -> 515b09a00


Changes needed to compile for cdh4-mr1


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/515b09a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/515b09a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/515b09a0

Branch: refs/heads/master
Commit: 515b09a002cbbb67c1ed22af90303a5f69135eb0
Parents: a8b67b3
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 15 15:57:56 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 15 15:57:56 2013 -0400

----------------------------------------------------------------------
 .../MasterBasedDistributedLayoutFactory.java    |  8 +--
 blur-gui/pom.xml                                | 19 +++++
 .../blur/mapreduce/lib/BlurOutputFormat.java    | 73 ++++++++++++--------
 .../blur/mapreduce/lib/CopyRateDirectory.java   |  8 ++-
 blur-thrift/pom.xml                             | 24 +++++++
 pom.xml                                         |  1 +
 6 files changed, 97 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/515b09a0/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
index e865da7..477619c 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
@@ -153,9 +153,9 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
       return newLayoutMap;
     } else {
       LOG.info("Gather counts for table [{0}]", table);
-      Collection<String> shardsThatAreOffline = new TreeSet<String>();
-      Map<String, Integer> onlineServerShardCount = new TreeMap<String, Integer>();
-      Map<String, String> existingLayoutMap = existingLayout.getLayout();
+      final Collection<String> shardsThatAreOffline = new TreeSet<String>();
+      final Map<String, Integer> onlineServerShardCount = new TreeMap<String, Integer>();
+      final Map<String, String> existingLayoutMap = existingLayout.getLayout();
       for (Entry<String, String> e : existingLayoutMap.entrySet()) {
         String shard = e.getKey();
         String server = e.getValue();
@@ -177,7 +177,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
 
       LOG.info("Assigning any missing shards [{1}] for table [{0}]", table, shardsThatAreOffline);
       // Assign missing shards
-      Map<String, String> newLayoutMap = new TreeMap<String, String>(existingLayoutMap);
+      final Map<String, String> newLayoutMap = new TreeMap<String, String>(existingLayoutMap);
       for (String offlineShard : shardsThatAreOffline) {
         // Find lowest shard count.
         String server = getServerWithTheLowest(onlineServerShardCount);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/515b09a0/blur-gui/pom.xml
----------------------------------------------------------------------
diff --git a/blur-gui/pom.xml b/blur-gui/pom.xml
index 21a07c5..d090ce8 100644
--- a/blur-gui/pom.xml
+++ b/blur-gui/pom.xml
@@ -131,4 +131,23 @@ under the License.
 			</plugins>
 		</pluginManagement>
 	</build>
+	
+	<profiles>
+		<profile>
+			<id>hadoop-1x</id>
+			<activation>
+				<activeByDefault>true</activeByDefault>
+			</activation>
+		</profile>
+		<profile>
+			<id>cdh4-mr1</id>
+			<dependencies>
+				<dependency>
+				    <groupId>org.mortbay.jetty</groupId>
+				    <artifactId>jetty</artifactId>
+				    <version>6.1.26</version>
+				</dependency>
+			</dependencies>
+		</profile>
+	</profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/515b09a0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index 2f59a20..f2e689d 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -359,16 +359,16 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
     private final Path _newIndex;
     private final boolean _indexLocally;
     private final boolean _optimizeInFlight;
-    private Counter _columnCount = emptyCounter();
-    private Counter _fieldCount = emptyCounter();
-    private Counter _recordCount = emptyCounter();
-    private Counter _rowCount = emptyCounter();
-    private Counter _recordDuplicateCount = emptyCounter();
-    private Counter _rowOverFlowCount = emptyCounter();
-    private Counter _rowDeleteCount = emptyCounter();
-    private RateCounter _recordRateCounter = new RateCounter(emptyCounter());
-    private RateCounter _rowRateCounter = new RateCounter(emptyCounter());
-    private RateCounter _copyRateCounter = new RateCounter(emptyCounter());
+    private Counter _columnCount;
+    private Counter _fieldCount;
+    private Counter _recordCount;
+    private Counter _rowCount;
+    private Counter _recordDuplicateCount;
+    private Counter _rowOverFlowCount;
+    private Counter _rowDeleteCount;
+    private RateCounter _recordRateCounter;
+    private RateCounter _rowRateCounter;
+    private RateCounter _copyRateCounter;
     private boolean _countersSetup = false;
     private IndexWriter _localTmpWriter;
     private boolean _usingLocalTmpindex;
@@ -417,11 +417,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
       }
     }
 
-    private Counter emptyCounter() {
-      return new Counter() {
-      };
-    }
-
     @Override
     public void write(Text key, BlurMutate value) throws IOException, InterruptedException
{
       if (!_countersSetup) {
@@ -457,14 +452,18 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
         _deletedRowId = blurRecord.getRowId();
         return;
       }
-      _columnCount.increment(record.getColumns().size());
+      if (_countersSetup) {
+        _columnCount.increment(record.getColumns().size());
+      }
       List<Field> document = TransactionRecorder.getDoc(_fieldManager, blurRecord.getRowId(),
record);
       List<Field> dup = _documents.put(recordId, document);
-      if (dup != null) {
-        _recordDuplicateCount.increment(1);
-      } else {
-        _fieldCount.increment(document.size());
-        _recordCount.increment(1);
+      if (_countersSetup) {
+        if (dup != null) {
+          _recordDuplicateCount.increment(1);
+        } else {
+          _fieldCount.increment(document.size());
+          _recordCount.increment(1);
+        }
       }
       flushToTmpIndexIfNeeded();
     }
@@ -522,28 +521,38 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
         flushToTmpIndex();
         _localTmpWriter.close(false);
         DirectoryReader reader = DirectoryReader.open(_localTmpDir);
-        _recordRateCounter.mark(reader.numDocs());
+        if (_countersSetup) {
+          _recordRateCounter.mark(reader.numDocs());
+        }
         _writer.addIndexes(reader);
         reader.close();
         resetLocalTmp();
-        _rowOverFlowCount.increment(1);
+        if (_countersSetup) {
+          _rowOverFlowCount.increment(1);
+        }
       } else {
         if (_documents.isEmpty()) {
           if (_deletedRowId != null) {
             _writer.addDocument(getDeleteDoc());
-            _rowDeleteCount.increment(1);
+            if (_countersSetup) {
+              _rowDeleteCount.increment(1);
+            }
           }
         } else {
           List<List<Field>> docs = new ArrayList<List<Field>>(_documents.values());
           docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
           _writer.addDocuments(docs);
-          _recordRateCounter.mark(_documents.size());
+          if (_countersSetup) {
+            _recordRateCounter.mark(_documents.size());
+          }
           _documents.clear();
         }
       }
       _deletedRowId = null;
-      _rowRateCounter.mark();
-      _rowCount.increment(1);
+      if (_countersSetup) {
+        _rowRateCounter.mark();
+        _rowCount.increment(1);
+      }
     }
 
     private Document getDeleteDoc() {
@@ -557,8 +566,10 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
     public void close(TaskAttemptContext context) throws IOException, InterruptedException
{
       flush();
       _writer.close();
-      _recordRateCounter.close();
-      _rowRateCounter.close();
+      if (_countersSetup) {
+        _recordRateCounter.close();
+        _rowRateCounter.close();
+      }
       if (_indexLocally) {
         if (_optimizeInFlight) {
           copyAndOptimizeInFlightDir();
@@ -566,7 +577,9 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
           copyDir();
         }
       }
-      _copyRateCounter.close();
+      if (_countersSetup) {
+        _copyRateCounter.close();
+      }
     }
 
     private void copyAndOptimizeInFlightDir() throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/515b09a0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java
index 886f2d6..a79541c 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java
@@ -58,12 +58,16 @@ public class CopyRateDirectory extends Directory {
 
     public void copyBytes(DataInput input, long numBytes) throws IOException {
       _indexOutput.copyBytes(input, numBytes);
-      _copyRateCounter.mark(numBytes);
+      if (_copyRateCounter != null) {
+        _copyRateCounter.mark(numBytes);
+      }
     }
 
     public void writeByte(byte b) throws IOException {
       _indexOutput.writeByte(b);
-      _copyRateCounter.mark();
+      if (_copyRateCounter != null) {
+        _copyRateCounter.mark();
+      }
     }
 
     public void flush() throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/515b09a0/blur-thrift/pom.xml
----------------------------------------------------------------------
diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml
index cdec767..890edae 100644
--- a/blur-thrift/pom.xml
+++ b/blur-thrift/pom.xml
@@ -100,4 +100,28 @@ under the License.
 			</plugin>
 		</plugins>
 	</build>
+	
+	
+
+	
+	<profiles>
+		<profile>
+			<id>hadoop-1x</id>
+			<activation>
+				<activeByDefault>true</activeByDefault>
+			</activation>
+		</profile>
+		<profile>
+			<id>cdh4-mr1</id>
+			<dependencies>
+				<dependency>
+				    <groupId>javax.servlet</groupId>
+				    <artifactId>servlet-api</artifactId>
+				    <version>${servlet-api.version}</version>
+				    <scope>provided</scope>
+				</dependency>
+			</dependencies>
+		</profile>
+	</profiles>
+	
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/515b09a0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 05e03b6..51f5f12 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,6 +189,7 @@ under the License.
 		<json.version>20090211</json.version>
 		<mrunit.version>0.9.0-incubating</mrunit.version>
 		<httpclient.version>4.1.3</httpclient.version>
+		<servlet-api.version>2.5</servlet-api.version>
 	</properties>
 
 	<modules>


Mime
View raw message