incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Making some pom changes and some code refactors so that the Blur can be compiled to run against hadoop 1.2.x and hadoop 2.2.x. Now to build on either plaform a mvn property switch is needed. -Dhadoop1 for hadoop 1.2.1 and -Dhadoop2 for hado
Date Mon, 24 Feb 2014 02:26:48 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/apache-blur-0.2 052c131e9 -> 2f0f94f2e


Making some pom changes and some code refactors so that the Blur can be compiled to run against
hadoop 1.2.x and hadoop 2.2.x.  Now to build on either plaform a mvn property switch is needed.
 -Dhadoop1 for hadoop 1.2.1 and -Dhadoop2 for hadoop 2.2.0


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2f0f94f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2f0f94f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2f0f94f2

Branch: refs/heads/apache-blur-0.2
Commit: 2f0f94f2e3bffa3a931fe421a1a726669c18fef6
Parents: 052c131
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sun Feb 23 21:26:38 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sun Feb 23 21:26:38 2014 -0500

----------------------------------------------------------------------
 blur-core/pom.xml                               | 22 ++++-----
 .../blur/manager/writer/IndexImporter.java      | 13 +++++-
 blur-gui/pom.xml                                | 11 ++++-
 blur-mapred/pom.xml                             | 22 ++++-----
 .../lib/BlurOutputFormatMiniClusterTest.java    |  4 +-
 .../mapreduce/lib/BlurOutputFormatTest.java     |  5 +-
 blur-store/pom.xml                              | 22 ++++-----
 .../blur/store/hdfs_v2/HdfsKeyValueStore.java   | 35 +++++++-------
 blur-store/src/test/resources/log4j.xml         | 48 ++++++++++++++++++++
 blur-thrift/pom.xml                             | 11 ++++-
 blur-util/pom.xml                               | 11 ++++-
 pom.xml                                         | 14 ++++--
 12 files changed, 146 insertions(+), 72 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-core/pom.xml
----------------------------------------------------------------------
diff --git a/blur-core/pom.xml b/blur-core/pom.xml
index 98cdcf8..dea4ccc 100644
--- a/blur-core/pom.xml
+++ b/blur-core/pom.xml
@@ -149,7 +149,9 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 			<dependencies>
 				<dependency>
@@ -161,7 +163,12 @@ under the License.
 			</dependencies>
 		</profile>
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<dependencies>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
@@ -170,17 +177,6 @@ under the License.
 				</dependency>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-core</artifactId>
-					<version>${hadoop.version}</version>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-test</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
 					<artifactId>hadoop-minicluster</artifactId>
 					<version>${hadoop.version}</version>
 					<scope>test</scope>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
index 0510f84..d8e9c44 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
@@ -26,6 +26,7 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -61,8 +62,10 @@ public class IndexImporter extends TimerTask implements Closeable {
   private final Timer _timer;
   private final String _table;
   private final String _shard;
+  private final AtomicBoolean _running = new AtomicBoolean();
 
   public IndexImporter(BlurIndex blurIndex, ShardContext shardContext, TimeUnit refreshUnit,
long refreshAmount) {
+    _running.set(true);
     _blurIndex = blurIndex;
     _shardContext = shardContext;
     _timer = new Timer("IndexImporter [" + shardContext.getShard() + "/" + shardContext.getTableContext().getTable()
@@ -75,8 +78,11 @@ public class IndexImporter extends TimerTask implements Closeable {
 
   @Override
   public void close() throws IOException {
-    _timer.cancel();
-    _timer.purge();
+    if (_running.get()) {
+      _running.set(false);
+      _timer.cancel();
+      _timer.purge();
+    }
   }
 
   @Override
@@ -87,6 +93,9 @@ public class IndexImporter extends TimerTask implements Closeable {
       FileSystem fileSystem = path.getFileSystem(configuration);
       SortedSet<FileStatus> listStatus;
       while (true) {
+        if (!_running.get()) {
+          return;
+        }
         try {
           listStatus = sort(fileSystem.listStatus(path, new PathFilter() {
             @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-gui/pom.xml
----------------------------------------------------------------------
diff --git a/blur-gui/pom.xml b/blur-gui/pom.xml
index 137c088..a3ce4df 100644
--- a/blur-gui/pom.xml
+++ b/blur-gui/pom.xml
@@ -136,11 +136,18 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 		</profile>
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<dependencies>
 				<dependency>
 				    <groupId>org.mortbay.jetty</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-mapred/pom.xml
----------------------------------------------------------------------
diff --git a/blur-mapred/pom.xml b/blur-mapred/pom.xml
index 7d50167..04041a3 100644
--- a/blur-mapred/pom.xml
+++ b/blur-mapred/pom.xml
@@ -131,7 +131,9 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 			<dependencies>
 				<dependency>
@@ -150,7 +152,12 @@ under the License.
 			</dependencies>
 		</profile>
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<dependencies>
 				<dependency>
 				      <groupId>org.apache.mrunit</groupId>
@@ -166,17 +173,6 @@ under the License.
 				</dependency>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-core</artifactId>
-					<version>${hadoop.version}</version>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-test</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
 					<artifactId>hadoop-minicluster</artifactId>
 					<version>${hadoop.version}</version>
 					<scope>test</scope>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
index f46a6ba..f470b35 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
@@ -46,8 +46,8 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.TestMapReduceLocal.TrackingTextInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -141,7 +141,7 @@ public class BlurOutputFormatMiniClusterTest {
     Job job = new Job(jobConf, "blur index");
     job.setJarByClass(BlurOutputFormatMiniClusterTest.class);
     job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
+    job.setInputFormatClass(TextInputFormat.class);
 
     FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
     String tableUri = new Path(TEST_ROOT_DIR + "/blur/" + tableName).toString();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
index 0666368..811dba5 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
@@ -43,8 +43,9 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.TestMapReduceLocal.TrackingTextInputFormat;
+import org.apache.hadoop.mapreduce.TestMapperReducerCleanup.TrackingTextInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.lucene.index.DirectoryReader;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -111,7 +112,7 @@ public class BlurOutputFormatTest {
     Job job = new Job(jobConf, "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
+    job.setInputFormatClass(TextInputFormat.class);
 
     FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
     String tableUri = new Path(TEST_ROOT_DIR + "/out").toString();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-store/pom.xml
----------------------------------------------------------------------
diff --git a/blur-store/pom.xml b/blur-store/pom.xml
index 35a9dea..9760d0c 100644
--- a/blur-store/pom.xml
+++ b/blur-store/pom.xml
@@ -140,7 +140,9 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 			<dependencies>
 				<dependency>
@@ -152,7 +154,12 @@ under the License.
 			</dependencies>
 		</profile>
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<dependencies>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
@@ -161,17 +168,6 @@ under the License.
 				</dependency>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-core</artifactId>
-					<version>${hadoop.version}</version>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-test</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
 					<artifactId>hadoop-minicluster</artifactId>
 					<version>${hadoop.version}</version>
 					<scope>test</scope>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java
b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java
index ec469f3..1841140 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java
@@ -21,6 +21,7 @@ import java.io.DataOutput;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -45,7 +46,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSClient.DFSInputStream;
 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Writable;
@@ -55,6 +55,8 @@ import org.apache.lucene.util.BytesRef;
 
 public class HdfsKeyValueStore implements Store {
 
+  private static final String IN = "in";
+  private static final String GET_FILE_LENGTH = "getFileLength";
   private static final int DEFAULT_MAX = 64 * 1024 * 1024;
   private final Log LOG = LogFactory.getLog(HdfsKeyValueStore.class);
 
@@ -370,8 +372,8 @@ public class HdfsKeyValueStore implements Store {
 
   private long getFileLength(Path path, FSDataInputStream inputStream) throws IOException
{
     FileStatus fileStatus = _fileSystem.getFileStatus(path);
-    DFSInputStream dfs = getDFS(inputStream);
-    return Math.max(dfs.getFileLength(), fileStatus.getLen());
+    long dfsLength = getDFSLength(inputStream);
+    return Math.max(dfsLength, fileStatus.getLen());
   }
 
   private void syncInternal() throws IOException {
@@ -417,25 +419,24 @@ public class HdfsKeyValueStore implements Store {
   }
 
   private SortedSet<FileStatus> getList(Path p) throws IOException {
-    FileStatus[] listStatus = _fileSystem.listStatus(p);
-    if (listStatus == null) {
-      return new TreeSet<FileStatus>();
+    if (_fileSystem.exists(p)) {
+      FileStatus[] listStatus = _fileSystem.listStatus(p);
+      if (listStatus != null) {
+        return new TreeSet<FileStatus>(Arrays.asList(listStatus));
+      }
     }
-    return new TreeSet<FileStatus>(Arrays.asList(listStatus));
+    return new TreeSet<FileStatus>();
   }
 
-  private static DFSInputStream getDFS(FSDataInputStream inputStream) throws IOException
{
+  private long getDFSLength(FSDataInputStream inputStream) throws IOException {
     try {
-      Field field = FilterInputStream.class.getDeclaredField("in");
+      Field field = FilterInputStream.class.getDeclaredField(IN);
       field.setAccessible(true);
-      return (DFSInputStream) field.get(inputStream);
-    } catch (NoSuchFieldException e) {
-      throw new IOException(e);
-    } catch (SecurityException e) {
-      throw new IOException(e);
-    } catch (IllegalArgumentException e) {
-      throw new IOException(e);
-    } catch (IllegalAccessException e) {
+      Object dfs = field.get(inputStream);
+      Method method = dfs.getClass().getMethod(GET_FILE_LENGTH, new Class[] {});
+      Object length = method.invoke(dfs, new Object[] {});
+      return (Long) length;
+    } catch (Exception e) {
       throw new IOException(e);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-store/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/blur-store/src/test/resources/log4j.xml b/blur-store/src/test/resources/log4j.xml
new file mode 100644
index 0000000..fad9ffa
--- /dev/null
+++ b/blur-store/src/test/resources/log4j.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+	<appender name="console" class="org.apache.log4j.ConsoleAppender">
+		<param name="Target" value="System.out" />
+		<layout class="org.apache.log4j.PatternLayout">
+			<param name="ConversionPattern" value="%-5p %d{yyyyMMdd_HH:mm:ss:SSS_z} [%t] %c{2}:
%m%n" />
+		</layout>
+	</appender>
+	<logger name="org.apache.hadoop">
+    	<level value="ERROR" />
+	    <appender-ref ref="console"/>
+	</logger>
+	<logger name="REQUEST_LOG" additivity="false">
+		<!-- Make value = "INFO"to enable -->
+    	<level value="ERROR" />
+	    <appender-ref ref="console"/>
+	</logger>
+	
+	<logger name="RESPONSE_LOG" additivity="false">
+		<!-- Make value = "INFO"to enable -->
+    	<level value="ERROR" />
+	    <appender-ref ref="console"/>
+	</logger>
+	<root>
+		<priority value="info" />
+		<appender-ref ref="console" />
+	</root>
+</log4j:configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-thrift/pom.xml
----------------------------------------------------------------------
diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml
index c812bc3..cc9a5d4 100644
--- a/blur-thrift/pom.xml
+++ b/blur-thrift/pom.xml
@@ -108,11 +108,18 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 		</profile>
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<dependencies>
 				<dependency>
 				    <groupId>javax.servlet</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-util/pom.xml
----------------------------------------------------------------------
diff --git a/blur-util/pom.xml b/blur-util/pom.xml
index 52dfcc6..faf45c7 100644
--- a/blur-util/pom.xml
+++ b/blur-util/pom.xml
@@ -192,7 +192,9 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 			<dependencies>
 				<dependency>
@@ -203,7 +205,12 @@ under the License.
 			</dependencies>
 		</profile>
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<dependencies>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9d114e1..bbb005a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -382,17 +382,23 @@ under the License.
 		<profile>
 			<id>hadoop-1x</id>
 			<activation>
-				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>hadoop1</name>
+				</property>
 			</activation>
 			<properties>
 				<hadoop.version>1.2.1</hadoop.version>
 			</properties>
 		</profile>
-
 		<profile>
-			<id>cdh4-mr1</id>
+			<id>hadoop-2.2</id>
+			<activation>
+				<property>
+					<name>hadoop2</name>
+				</property>
+			</activation>
 			<properties>
-				<hadoop.version>2.0.0-mr1-cdh4.3.0</hadoop.version>
+				<hadoop.version>2.2.0</hadoop.version>
 			</properties>
 		</profile>
 	</profiles>


Mime
View raw message