incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: removing final files from the seperate hadoop1 and 2 mapred projects.
Date Thu, 16 Apr 2015 00:49:48 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 578130245 -> e0e40d551


removing final files from the seperate hadoop1 and 2 mapred projects.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e0e40d55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e0e40d55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e0e40d55

Branch: refs/heads/master
Commit: e0e40d551287074c441c82628bc968a5329ca167
Parents: 5781302
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Apr 15 20:36:42 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Apr 15 20:36:42 2015 -0400

----------------------------------------------------------------------
 blur-mapred-hadoop1/pom.xml                     | 223 ---------
 .../lib/BlurOutputFormatMiniClusterTest.java    | 236 ----------
 .../mapreduce/lib/BlurOutputFormatTest.java     | 463 ------------------
 blur-mapred-hadoop2/pom.xml                     | 178 -------
 .../lib/BlurOutputFormatMiniClusterTest.java    | 241 ----------
 .../mapreduce/lib/BlurOutputFormatTest.java     | 465 -------------------
 6 files changed, 1806 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e0e40d55/blur-mapred-hadoop1/pom.xml
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/pom.xml b/blur-mapred-hadoop1/pom.xml
deleted file mode 100644
index c573e2f..0000000
--- a/blur-mapred-hadoop1/pom.xml
+++ /dev/null
@@ -1,223 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<parent>
-		<groupId>org.apache.blur</groupId>
-		<artifactId>blur</artifactId>
-		<version>0.2.4-incubating-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
-	</parent>
-	<groupId>org.apache.blur</groupId>
-	<artifactId>blur-mapred-hadoop1</artifactId>
-	<version>${projectVersion}</version>
-	<packaging>jar</packaging>
-	<name>Blur Map Reduce Hadoop1</name>
-	<description>The Blur Map Reduce Hadoop1 module contains the testsuite for Hadoop1.</description>
-
-	<dependencies>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-mapred-common</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.zookeeper</groupId>
-			<artifactId>zookeeper</artifactId>
-			<version>${zookeeper.version}</version>
-			<scope>provided</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-core</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-core</artifactId>
-			<version>${project.version}</version>
-			<type>test-jar</type>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-store</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-util</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-util</artifactId>
-			<version>${project.version}</version>
-			<type>test-jar</type>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>${log4j.version}</version>
-			<scope>provided</scope>
-			<exclusions>
-				<exclusion>
-					<groupId>javax.mail</groupId>
-					<artifactId>mail</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>javax.jms</groupId>
-					<artifactId>jms</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>com.sun.jdmk</groupId>
-					<artifactId>jmxtools</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>com.sun.jmx</groupId>
-					<artifactId>jmxri</artifactId>
-				</exclusion>
-			</exclusions>
-		</dependency>
-	</dependencies>
-
-	<repositories>
-		<repository>
-			<id>libdir</id>
-			<url>file://${basedir}/../lib</url>
-		</repository>
-	</repositories>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-jar-plugin</artifactId>
-				<executions>
-					<execution>
-						<goals>
-							<goal>test-jar</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-help-plugin</artifactId>
-				<version>2.2</version>
-				<executions>
-					<execution>
-						<phase>generate-resources</phase>
-						<goals>
-							<goal>effective-pom</goal>
-						</goals>
-						<configuration>
-							<output>${project.build.directory}/effective-pom.xml</output>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-install-plugin</artifactId>
-				<version>2.3.1</version>
-				<executions>
-					<execution>
-						<phase>install</phase>
-						<goals>
-							<goal>install-file</goal>
-						</goals>
-						<configuration>
-							<file>${project.build.directory}/${artifactId}-${project.version}.jar</file>
-							<pomFile>${project.build.directory}/effective-pom.xml</pomFile>
-							<!-- sources></sources -->
-							<!-- javadoc></javadoc -->
-							<groupId>${project.groupId}</groupId>
-							<artifactId>${project.artifactId}</artifactId>
-							<version>${project.version}</version>
-							<packaging>jar</packaging>
-							<!--classifier></classifier -->
-							<generatePom>true</generatePom>
-							<createChecksum>true</createChecksum>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-		</plugins>
-	</build>
-	
-	
-	<profiles>
-		<profile>
-			<id>hadoop1</id>
-			<activation>
-				<property>
-					<name>hadoop1</name>
-				</property>
-			</activation>
-			<dependencies>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-test</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
-				<dependency>
-				      <groupId>org.apache.mrunit</groupId>
-				      <artifactId>mrunit</artifactId>
-				      <version>${mrunit.version}</version>
-				      <classifier>hadoop1</classifier>
-					  <scope>test</scope>
-                </dependency>
-			</dependencies>
-		</profile>
-		<profile>
-			<id>hadoop2-mr1</id>
-			<activation>
-				<property>
-					<name>hadoop2-mr1</name>
-				</property>
-			</activation>
-			<dependencies>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-test</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
-                <dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-minicluster</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
-				<dependency>
-				      <groupId>org.apache.mrunit</groupId>
-				      <artifactId>mrunit</artifactId>
-				      <version>${mrunit.version}</version>
-				      <classifier>hadoop2</classifier>
-					  <scope>test</scope>
-                </dependency>
-			</dependencies>
-		</profile>
-	</profiles>
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e0e40d55/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
deleted file mode 100644
index c5e2562..0000000
--- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
+++ /dev/null
@@ -1,236 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.server.TableContext;
-import org.apache.blur.store.buffer.BufferStore;
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.BlurClient;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.TableStats;
-import org.apache.blur.utils.GCWatcher;
-import org.apache.blur.utils.ShardUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class BlurOutputFormatMiniClusterTest {
-
-  private static Configuration conf = new Configuration();
-  private static FileSystem fileSystem;
-  private static MiniMRCluster mr;
-  private static Path TEST_ROOT_DIR;
-  private static JobConf jobConf;
-  private static MiniCluster miniCluster;
-  private Path inDir = new Path(TEST_ROOT_DIR + "/in");
-  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir",
-      "./target/tmp_BlurOutputFormatMiniClusterTest"));
-
-  @BeforeClass
-  public static void setupTest() throws Exception {
-    GCWatcher.init(0.60);
-    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
-    File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
-    testDirectory.mkdirs();
-
-    Path directory = new Path(testDirectory.getPath());
-    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
-    FsAction userAction = dirPermissions.getUserAction();
-    FsAction groupAction = dirPermissions.getGroupAction();
-    FsAction otherAction = dirPermissions.getOtherAction();
-
-    StringBuilder builder = new StringBuilder();
-    builder.append(userAction.ordinal());
-    builder.append(groupAction.ordinal());
-    builder.append(otherAction.ordinal());
-    String dirPermissionNum = builder.toString();
-    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
-    testDirectory.delete();
-    miniCluster = new MiniCluster();
-    miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, false);
-
-    // System.setProperty("test.build.data",
-    // "./target/BlurOutputFormatTest/data");
-    // TEST_ROOT_DIR = new Path(System.getProperty("test.build.data",
-    // "target/tmp/BlurOutputFormatTest_tmp"));
-    TEST_ROOT_DIR = new Path(miniCluster.getFileSystemUri().toString() + "/blur_test");
-    System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
-    try {
-      fileSystem = TEST_ROOT_DIR.getFileSystem(conf);
-    } catch (IOException io) {
-      throw new RuntimeException("problem getting local fs", io);
-    }
-    mr = new MiniMRCluster(1, miniCluster.getFileSystemUri().toString(), 1);
-    jobConf = mr.createJobConf();
-    BufferStore.initNewBuffer(128, 128 * 128);
-  }
-
-  @AfterClass
-  public static void teardown() {
-    if (mr != null) {
-      mr.shutdown();
-    }
-    miniCluster.shutdownBlurCluster();
-    rm(new File("build"));
-  }
-
-  private static void rm(File file) {
-    if (!file.exists()) {
-      return;
-    }
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-  @Before
-  public void setup() {
-    TableContext.clear();
-  }
-
-  @Test
-  public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException, BlurException,
-      TException {
-    fileSystem.delete(inDir, true);
-    String tableName = "testBlurOutputFormat";
-    writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
-    writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
-
-    Job job = new Job(jobConf, "blur index");
-    job.setJarByClass(BlurOutputFormatMiniClusterTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    String tableUri = new Path(TEST_ROOT_DIR + "/blur/" + tableName).toString();
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(tableUri);
-    tableDescriptor.setName(tableName);
-
-    Iface client = getClient();
-    client.createTable(tableDescriptor);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-
-    Path tablePath = new Path(tableUri);
-    Path shardPath = new Path(tablePath, ShardUtil.getShardName(0));
-    FileStatus[] listStatus = fileSystem.listStatus(shardPath);
-    assertEquals(3, listStatus.length);
-    System.out.println("======" + listStatus.length);
-    for (FileStatus fileStatus : listStatus) {
-      System.out.println(fileStatus.getPath());
-    }
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    client.loadData(tableName, output.toString());
-
-    while (true) {
-      TableStats tableStats = client.tableStats(tableName);
-      System.out.println(tableStats);
-      if (tableStats.getRowCount() > 0) {
-        break;
-      }
-      Thread.sleep(5000);
-    }
-
-    assertTrue(fileSystem.exists(tablePath));
-    assertFalse(fileSystem.isFile(tablePath));
-
-    FileStatus[] listStatusAfter = fileSystem.listStatus(shardPath);
-
-    assertEquals(11, listStatusAfter.length);
-
-  }
-
-  private Iface getClient() {
-    return BlurClient.getClient(miniCluster.getControllerConnectionStr());
-  }
-
-  public static String readFile(String name) throws IOException {
-    DataInputStream f = fileSystem.open(new Path(TEST_ROOT_DIR + "/" + name));
-    BufferedReader b = new BufferedReader(new InputStreamReader(f));
-    StringBuilder result = new StringBuilder();
-    String line = b.readLine();
-    while (line != null) {
-      result.append(line);
-      result.append('\n');
-      line = b.readLine();
-    }
-    b.close();
-    return result.toString();
-  }
-
-  private Path writeRecordsFile(String name, int starintgRowId, int numberOfRows, int startRecordId,
-      int numberOfRecords, String family) throws IOException {
-    // "1,1,cf1,val1"
-    Path file = new Path(TEST_ROOT_DIR + "/" + name);
-    fileSystem.delete(file, false);
-    DataOutputStream f = fileSystem.create(file);
-    PrintWriter writer = new PrintWriter(f);
-    for (int row = 0; row < numberOfRows; row++) {
-      for (int record = 0; record < numberOfRecords; record++) {
-        writer.println(getRecord(row + starintgRowId, record + startRecordId, family));
-      }
-    }
-    writer.close();
-    return file;
-  }
-
-  private String getRecord(int rowId, int recordId, String family) {
-    return rowId + "," + recordId + "," + family + ",valuetoindex";
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e0e40d55/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
deleted file mode 100644
index 03364b1..0000000
--- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ /dev/null
@@ -1,463 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.util.Collection;
-import java.util.List;
-import java.util.TreeSet;
-
-import org.apache.blur.server.TableContext;
-import org.apache.blur.store.buffer.BufferStore;
-import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.ShardUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.BytesRef;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class BlurOutputFormatTest {
-
-  private static Configuration conf = new Configuration();
-  private static FileSystem localFs;
-  private static MiniMRCluster mr;
-  private static final Path TEST_ROOT_DIR = new Path("./target/tmp/BlurOutputFormatTest_tmp");
-  private static JobConf jobConf;
-  private static final Path outDir = new Path(TEST_ROOT_DIR + "/out");
-  private static final Path inDir = new Path(TEST_ROOT_DIR + "/in");
-
-  @BeforeClass
-  public static void setupTest() throws Exception {
-    System.setProperty("test.build.data", "./target/BlurOutputFormatTest/data");
-    System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
-    try {
-      localFs = FileSystem.getLocal(conf);
-    } catch (IOException io) {
-      throw new RuntimeException("problem getting local fs", io);
-    }
-    mr = new MiniMRCluster(1, "file:///", 1);
-    jobConf = mr.createJobConf();
-    BufferStore.initNewBuffer(128, 128 * 128);
-  }
-
-  @AfterClass
-  public static void teardown() {
-    if (mr != null) {
-      mr.shutdown();
-    }
-    rm(new File("build"));
-  }
-
-  private static void rm(File file) {
-    if (!file.exists()) {
-      return;
-    }
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-  @Before
-  public void setup() throws IllegalArgumentException, IOException {
-    TableContext.clear();
-    if (localFs.exists(inDir)) {
-      assertTrue(localFs.delete(inDir, true));
-    }
-    if (localFs.exists(outDir)) {
-      assertTrue(localFs.delete(outDir, true));
-    }
-  }
-
-  @Test
-  public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException {
-    writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
-    writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
-
-    Job job = new Job(new Configuration(jobConf), "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 1);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    Path path = new Path(output, ShardUtil.getShardName(0));
-    Collection<Path> commitedTasks = getCommitedTasks(path);
-    assertEquals(1, commitedTasks.size());
-    DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
-    assertEquals(2, reader.numDocs());
-    validatePrimeDocs(reader);
-    reader.close();
-  }
-
-  private Collection<Path> getCommitedTasks(Path path) throws IOException {
-    Collection<Path> result = new TreeSet<Path>();
-    FileSystem fileSystem = path.getFileSystem(jobConf);
-    FileStatus[] listStatus = fileSystem.listStatus(path);
-    for (FileStatus fileStatus : listStatus) {
-      Path p = fileStatus.getPath();
-      if (fileStatus.isDir() && p.getName().endsWith(".commit")) {
-        result.add(p);
-      }
-    }
-    return result;
-  }
-
-  @Test
-  public void testBlurOutputFormatOverFlowTest() throws IOException, InterruptedException, ClassNotFoundException {
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
-
-    Job job = new Job(new Configuration(jobConf), "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 1);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setIndexLocally(job, true);
-    BlurOutputFormat.setOptimizeInFlight(job, false);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    Path path = new Path(output, ShardUtil.getShardName(0));
-    Collection<Path> commitedTasks = getCommitedTasks(path);
-    assertEquals(1, commitedTasks.size());
-
-    DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
-    assertEquals(80000, reader.numDocs());
-    validatePrimeDocs(reader);
-    reader.close();
-  }
-
-  private void validatePrimeDocs(DirectoryReader reader) throws IOException {
-    List<AtomicReaderContext> leaves = reader.leaves();
-    for (AtomicReaderContext context : leaves) {
-      AtomicReader atomicReader = context.reader();
-      Terms rowIdTerms = atomicReader.fields().terms("rowid");
-
-      TermsEnum rowIdTermsEnum = rowIdTerms.iterator(null);
-      BytesRef rowId;
-      while ((rowId = rowIdTermsEnum.next()) != null) {
-        DocsEnum rowIdDocsEnum = rowIdTermsEnum.docs(atomicReader.getLiveDocs(), null);
-        int nextDoc = rowIdDocsEnum.nextDoc();
-        checkPrimeDoc(atomicReader, nextDoc, rowId);
-      }
-    }
-  }
-
-  private void checkPrimeDoc(AtomicReader atomicReader, int docId, BytesRef rowId) throws IOException {
-    Terms primeDocTerms = atomicReader.fields().terms("_prime_");
-    TermsEnum primeDocTermsEnum = primeDocTerms.iterator(null);
-    if (!primeDocTermsEnum.seekExact(new BytesRef("true"), false)) {
-      fail("No Prime Docs...");
-    }
-    DocsEnum primeDocDocsEnum = primeDocTermsEnum.docs(atomicReader.getLiveDocs(), null);
-    int advance;
-    if ((advance = primeDocDocsEnum.advance(docId)) != docId) {
-      fail("FAIL:" + rowId.utf8ToString() + " " + advance + " " + docId);
-    }
-  }
-
-  @Test
-  public void testBlurOutputFormatOverFlowMultipleReducersTest() throws IOException, InterruptedException,
-      ClassNotFoundException {
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
-
-    Job job = new Job(new Configuration(jobConf), "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(2);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 2);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setIndexLocally(job, false);
-    BlurOutputFormat.setDocumentBufferStrategy(job, DocumentBufferStrategyHeapSize.class);
-    BlurOutputFormat.setMaxDocumentBufferHeapSize(job, 128 * 1024);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    long total = 0;
-    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
-      Path path = new Path(output, ShardUtil.getShardName(i));
-      Collection<Path> commitedTasks = getCommitedTasks(path);
-      assertEquals(1, commitedTasks.size());
-
-      DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
-      total += reader.numDocs();
-      validatePrimeDocs(reader);
-      reader.close();
-    }
-    assertEquals(80000, total);
-
-  }
-
-  @Test
-  public void testBlurOutputFormatOverFlowMultipleReducersWithReduceMultiplierTest() throws IOException,
-      InterruptedException, ClassNotFoundException {
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
-
-    Job job = new Job(new Configuration(jobConf), "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(7);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 7);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    int multiple = 2;
-    BlurOutputFormat.setReducerMultiplier(job, multiple);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    long total = 0;
-    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
-      Path path = new Path(output, ShardUtil.getShardName(i));
-      Collection<Path> commitedTasks = getCommitedTasks(path);
-      assertTrue(multiple >= commitedTasks.size());
-      for (Path p : commitedTasks) {
-        DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, p));
-        total += reader.numDocs();
-        validatePrimeDocs(reader);
-        reader.close();
-      }
-    }
-    assertEquals(80000, total);
-
-  }
-
-  @Test(expected = IllegalArgumentException.class)
-  public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException,
-      ClassNotFoundException {
-    writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
-    writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
-
-    Job job = new Job(new Configuration(jobConf), "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 1);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setReducerMultiplier(job, 2);
-    job.setNumReduceTasks(4);
-    job.submit();
-
-  }
-
-  // @TODO this test to fail sometimes due to issues in the MR MiniCluster
-  // @Test
-  public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException,
-      ClassNotFoundException {
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 5000, 2000, 100, "cf1"); // 100 * 5000 =
-                                                             // 500,000
-
-    Job job = new Job(new Configuration(jobConf), "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    String tableUri = new Path(TEST_ROOT_DIR + "/out").toString();
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(2);
-    tableDescriptor.setTableUri(tableUri);
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 2);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    BlurOutputFormat.setIndexLocally(job, false);
-
-    job.submit();
-    boolean killCalled = false;
-    while (!job.isComplete()) {
-      Thread.sleep(1000);
-      System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100,
-          job.reduceProgress() * 100);
-      if (job.reduceProgress() > 0.7 && !killCalled) {
-        job.killJob();
-        killCalled = true;
-      }
-    }
-
-    assertFalse(job.isSuccessful());
-
-    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
-      Path path = new Path(tableUri, ShardUtil.getShardName(i));
-      FileSystem fileSystem = path.getFileSystem(job.getConfiguration());
-      FileStatus[] listStatus = fileSystem.listStatus(path);
-      assertEquals(toString(listStatus), 0, listStatus.length);
-    }
-  }
-
-  private String toString(FileStatus[] listStatus) {
-    if (listStatus == null || listStatus.length == 0) {
-      return "";
-    }
-    String s = "";
-    for (FileStatus fileStatus : listStatus) {
-      if (s.length() > 0) {
-        s += ",";
-      }
-      s += fileStatus.getPath();
-    }
-    return s;
-  }
-
-  public static String readFile(String name) throws IOException {
-    DataInputStream f = localFs.open(new Path(TEST_ROOT_DIR + "/" + name));
-    BufferedReader b = new BufferedReader(new InputStreamReader(f));
-    StringBuilder result = new StringBuilder();
-    String line = b.readLine();
-    while (line != null) {
-      result.append(line);
-      result.append('\n');
-      line = b.readLine();
-    }
-    b.close();
-    return result.toString();
-  }
-
-  private Path writeRecordsFile(String name, int starintgRowId, int numberOfRows, int startRecordId,
-      int numberOfRecords, String family) throws IOException {
-    // "1,1,cf1,val1"
-    Path file = new Path(TEST_ROOT_DIR + "/" + name);
-    localFs.delete(file, false);
-    DataOutputStream f = localFs.create(file);
-    PrintWriter writer = new PrintWriter(f);
-    for (int row = 0; row < numberOfRows; row++) {
-      for (int record = 0; record < numberOfRecords; record++) {
-        writer.println(getRecord(row + starintgRowId, record + startRecordId, family));
-      }
-    }
-    writer.close();
-    return file;
-  }
-
-  private void createShardDirectories(Path outDir, int shardCount) throws IOException {
-    localFs.mkdirs(outDir);
-    for (int i = 0; i < shardCount; i++) {
-      localFs.mkdirs(new Path(outDir, ShardUtil.getShardName(i)));
-    }
-  }
-
-  private String getRecord(int rowId, int recordId, String family) {
-    return rowId + "," + recordId + "," + family + ",valuetoindex";
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e0e40d55/blur-mapred-hadoop2/pom.xml
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/pom.xml b/blur-mapred-hadoop2/pom.xml
deleted file mode 100644
index 4b4720c..0000000
--- a/blur-mapred-hadoop2/pom.xml
+++ /dev/null
@@ -1,178 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<parent>
-		<groupId>org.apache.blur</groupId>
-		<artifactId>blur</artifactId>
-		<version>0.2.4-incubating-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
-	</parent>
-	<groupId>org.apache.blur</groupId>
-	<artifactId>blur-mapred-hadoop2</artifactId>
-	<version>${projectVersion}</version>
-	<packaging>jar</packaging>
-	<name>Blur Map Reduce Hadoop2</name>
-	<description>The Blur Map Reduce Hadoop2 module contains the testsuite for Hadoop2.</description>
-
-	<dependencies>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-mapred-common</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.zookeeper</groupId>
-			<artifactId>zookeeper</artifactId>
-			<version>${zookeeper.version}</version>
-			<scope>provided</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-core</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-core</artifactId>
-			<version>${project.version}</version>
-			<type>test-jar</type>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-store</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-util</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.blur</groupId>
-			<artifactId>blur-util</artifactId>
-			<version>${project.version}</version>
-			<type>test-jar</type>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<scope>provided</scope>
-		</dependency>
-	</dependencies>
-
-	<repositories>
-		<repository>
-			<id>libdir</id>
-			<url>file://${basedir}/../lib</url>
-		</repository>
-	</repositories>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-jar-plugin</artifactId>
-				<executions>
-					<execution>
-						<goals>
-							<goal>test-jar</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-help-plugin</artifactId>
-				<version>2.2</version>
-				<executions>
-					<execution>
-						<phase>generate-resources</phase>
-						<goals>
-							<goal>effective-pom</goal>
-						</goals>
-						<configuration>
-							<output>${project.build.directory}/effective-pom.xml</output>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-install-plugin</artifactId>
-				<version>2.3.1</version>
-				<executions>
-					<execution>
-						<phase>install</phase>
-						<goals>
-							<goal>install-file</goal>
-						</goals>
-						<configuration>
-							<file>${project.build.directory}/${artifactId}-${project.version}.jar</file>
-							<pomFile>${project.build.directory}/effective-pom.xml</pomFile>
-							<!-- sources></sources -->
-							<!-- javadoc></javadoc -->
-							<groupId>${project.groupId}</groupId>
-							<artifactId>${project.artifactId}</artifactId>
-							<version>${project.version}</version>
-							<packaging>jar</packaging>
-							<!--classifier></classifier -->
-							<generatePom>true</generatePom>
-							<createChecksum>true</createChecksum>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-		</plugins>
-	</build>
-	
-	
-	<profiles>
-		<profile>
-			<id>hadoop2</id>
-			<activation>
-				<property>
-					<name>hadoop2</name>
-				</property>
-			</activation>
-			<dependencies>
-				<dependency>
-				      <groupId>org.apache.mrunit</groupId>
-				      <artifactId>mrunit</artifactId>
-				      <version>${mrunit.version}</version>
-				      <classifier>hadoop2</classifier>
-					  <scope>test</scope>
-                </dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-client</artifactId>
-				</dependency>
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-minicluster</artifactId>
-					<scope>test</scope>
-				</dependency>
-			</dependencies>
-		</profile>
-	</profiles>
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e0e40d55/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
deleted file mode 100644
index 5fdc9ab..0000000
--- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.server.TableContext;
-import org.apache.blur.store.buffer.BufferStore;
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.BlurClient;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.TableStats;
-import org.apache.blur.utils.GCWatcher;
-import org.apache.blur.utils.ShardUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.mapred.MiniMRClientClusterFactory;
-import org.apache.hadoop.mapred.MiniMRYarnClusterAdapter;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class BlurOutputFormatMiniClusterTest {
-
-  private static Configuration conf = new Configuration();
-  private static FileSystem fileSystem;
-  private static MiniMRYarnClusterAdapter mr;
-  private static Path TEST_ROOT_DIR;
-  private static MiniCluster miniCluster;
-  private Path inDir = new Path(TEST_ROOT_DIR + "/in");
-  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir",
-      "./target/tmp_BlurOutputFormatMiniClusterTest"));
-
-  @BeforeClass
-  public static void setupTest() throws Exception {
-    GCWatcher.init(0.60);
-    BlurOutputFormatTest.setupJavaHome();
-    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
-    File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
-    testDirectory.mkdirs();
-
-    Path directory = new Path(testDirectory.getPath());
-    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
-    FsAction userAction = dirPermissions.getUserAction();
-    FsAction groupAction = dirPermissions.getGroupAction();
-    FsAction otherAction = dirPermissions.getOtherAction();
-
-    StringBuilder builder = new StringBuilder();
-    builder.append(userAction.ordinal());
-    builder.append(groupAction.ordinal());
-    builder.append(otherAction.ordinal());
-    String dirPermissionNum = builder.toString();
-    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
-    testDirectory.delete();
-    miniCluster = new MiniCluster();
-    miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, false);
-
-    // System.setProperty("test.build.data",
-    // "./target/BlurOutputFormatTest/data");
-    // TEST_ROOT_DIR = new Path(System.getProperty("test.build.data",
-    // "target/tmp/BlurOutputFormatTest_tmp"));
-    TEST_ROOT_DIR = new Path(miniCluster.getFileSystemUri().toString() + "/blur_test");
-    System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
-    try {
-      fileSystem = TEST_ROOT_DIR.getFileSystem(conf);
-    } catch (IOException io) {
-      throw new RuntimeException("problem getting local fs", io);
-    }
-
-    FileSystem.setDefaultUri(conf, miniCluster.getFileSystemUri());
-    mr = (MiniMRYarnClusterAdapter) MiniMRClientClusterFactory.create(BlurOutputFormatTest.class, 1, conf);
-    mr.start();
-    conf = mr.getConfig();
-
-    BufferStore.initNewBuffer(128, 128 * 128);
-  }
-
-  @AfterClass
-  public static void teardown() {
-    if (mr != null) {
-      mr.stop();
-    }
-    miniCluster.shutdownBlurCluster();
-    rm(new File("build"));
-  }
-
-  private static void rm(File file) {
-    if (!file.exists()) {
-      return;
-    }
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-  @Before
-  public void setup() {
-    TableContext.clear();
-  }
-
-  @Test
-  public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException, BlurException,
-      TException {
-    fileSystem.delete(inDir, true);
-    String tableName = "testBlurOutputFormat";
-    writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
-    writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatMiniClusterTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    String tableUri = new Path(TEST_ROOT_DIR + "/blur/" + tableName).makeQualified(fileSystem.getUri(),
-        fileSystem.getWorkingDirectory()).toString();
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(tableUri);
-    tableDescriptor.setName(tableName);
-
-    Iface client = getClient();
-    client.createTable(tableDescriptor);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-
-    Path tablePath = new Path(tableUri);
-    Path shardPath = new Path(tablePath, ShardUtil.getShardName(0));
-    FileStatus[] listStatus = fileSystem.listStatus(shardPath);
-    assertEquals(3, listStatus.length);
-    System.out.println("======" + listStatus.length);
-    for (FileStatus fileStatus : listStatus) {
-      System.out.println(fileStatus.getPath());
-    }
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    client.loadData(tableName, output.toString());
-
-    while (true) {
-      TableStats tableStats = client.tableStats(tableName);
-      System.out.println(tableStats);
-      if (tableStats.getRowCount() > 0) {
-        break;
-      }
-      Thread.sleep(100);
-    }
-
-    assertTrue(fileSystem.exists(tablePath));
-    assertFalse(fileSystem.isFile(tablePath));
-
-    FileStatus[] listStatusAfter = fileSystem.listStatus(shardPath);
-
-    assertEquals(11, listStatusAfter.length);
-
-  }
-
-  private Iface getClient() {
-    return BlurClient.getClient(miniCluster.getControllerConnectionStr());
-  }
-
-  public static String readFile(String name) throws IOException {
-    DataInputStream f = fileSystem.open(new Path(TEST_ROOT_DIR + "/" + name));
-    BufferedReader b = new BufferedReader(new InputStreamReader(f));
-    StringBuilder result = new StringBuilder();
-    String line = b.readLine();
-    while (line != null) {
-      result.append(line);
-      result.append('\n');
-      line = b.readLine();
-    }
-    b.close();
-    return result.toString();
-  }
-
-  private Path writeRecordsFile(String name, int starintgRowId, int numberOfRows, int startRecordId,
-      int numberOfRecords, String family) throws IOException {
-    // "1,1,cf1,val1"
-    Path file = new Path(TEST_ROOT_DIR + "/" + name);
-    fileSystem.delete(file, false);
-    DataOutputStream f = fileSystem.create(file);
-    PrintWriter writer = new PrintWriter(f);
-    for (int row = 0; row < numberOfRows; row++) {
-      for (int record = 0; record < numberOfRecords; record++) {
-        writer.println(getRecord(row + starintgRowId, record + startRecordId, family));
-      }
-    }
-    writer.close();
-    return file;
-  }
-
-  private String getRecord(int rowId, int recordId, String family) {
-    return rowId + "," + recordId + "," + family + ",valuetoindex";
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e0e40d55/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
deleted file mode 100644
index 5de9231..0000000
--- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ /dev/null
@@ -1,465 +0,0 @@
-package org.apache.blur.mapreduce.lib;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.net.URI;
-import java.util.Collection;
-import java.util.TreeSet;
-
-import org.apache.blur.server.TableContext;
-import org.apache.blur.store.buffer.BufferStore;
-import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.ShardUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.MiniMRClientClusterFactory;
-import org.apache.hadoop.mapred.MiniMRYarnClusterAdapter;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.TestMapperReducerCleanup.TrackingTextInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.lucene.index.DirectoryReader;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class BlurOutputFormatTest {
-
-  private static Configuration conf = new Configuration();
-  private static FileSystem localFs;
-  private static Path TEST_ROOT_DIR;
-  private static MiniMRYarnClusterAdapter mr;
-  private Path outDir = new Path(TEST_ROOT_DIR + "/out");
-  private Path inDir = new Path(TEST_ROOT_DIR + "/in");
-
-  @BeforeClass
-  public static void setupTest() throws Exception {
-    setupJavaHome();
-    File file = new File("./target/tmp/BlurOutputFormatTest_tmp");
-    String pathStr = file.getAbsoluteFile().toURI().toString();
-    System.setProperty("test.build.data", pathStr + "/data");
-    System.setProperty("hadoop.log.dir", pathStr + "/hadoop_log");
-    try {
-      localFs = FileSystem.getLocal(conf);
-    } catch (IOException io) {
-      throw new RuntimeException("problem getting local fs", io);
-    }
-    TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", pathStr));
-
-    FileSystem.setDefaultUri(conf, new URI("file:///"));
-    mr = (MiniMRYarnClusterAdapter) MiniMRClientClusterFactory.create(BlurOutputFormatTest.class, 1, conf);
-    mr.start();
-    conf = mr.getConfig();
-
-    BufferStore.initNewBuffer(128, 128 * 128);
-  }
-
-  public static void setupJavaHome() {
-    String str = System.getenv("JAVA_HOME");
-    if (str == null) {
-      String property = System.getProperty("java.home");
-      if (property != null) {
-        throw new RuntimeException("JAVA_HOME not set should probably be [" + property + "].");
-      }
-      throw new RuntimeException("JAVA_HOME not set.");
-    }
-  }
-
-  @AfterClass
-  public static void teardown() throws IOException {
-    if (mr != null) {
-      mr.stop();
-    }
-    rm(new File("build"));
-  }
-
-  private static void rm(File file) {
-    if (!file.exists()) {
-      return;
-    }
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-  @Before
-  public void setup() {
-    TableContext.clear();
-  }
-
-  @Test
-  public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException {
-    localFs.delete(inDir, true);
-    localFs.delete(outDir, true);
-    writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
-    writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 1);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    Path path = new Path(output, ShardUtil.getShardName(0));
-    dump(path, conf);
-    Collection<Path> commitedTasks = getCommitedTasks(path);
-    assertEquals(1, commitedTasks.size());
-    DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
-    assertEquals(2, reader.numDocs());
-    reader.close();
-  }
-
-  private void dump(Path path, Configuration conf) throws IOException {
-    FileSystem fileSystem = path.getFileSystem(conf);
-    System.out.println(path);
-    if (!fileSystem.isFile(path)) {
-      FileStatus[] listStatus = fileSystem.listStatus(path);
-      for (FileStatus fileStatus : listStatus) {
-        dump(fileStatus.getPath(), conf);
-      }
-    }
-  }
-
-  private Collection<Path> getCommitedTasks(Path path) throws IOException {
-    Collection<Path> result = new TreeSet<Path>();
-    FileSystem fileSystem = path.getFileSystem(conf);
-    FileStatus[] listStatus = fileSystem.listStatus(path);
-    for (FileStatus fileStatus : listStatus) {
-      Path p = fileStatus.getPath();
-      if (fileStatus.isDirectory() && p.getName().endsWith(".commit")) {
-        result.add(p);
-      }
-    }
-    return result;
-  }
-
-  @Test
-  public void testBlurOutputFormatOverFlowTest() throws IOException, InterruptedException, ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 1);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setIndexLocally(job, true);
-    BlurOutputFormat.setOptimizeInFlight(job, false);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    Path path = new Path(output, ShardUtil.getShardName(0));
-    Collection<Path> commitedTasks = getCommitedTasks(path);
-    assertEquals(1, commitedTasks.size());
-
-    DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
-    assertEquals(80000, reader.numDocs());
-    reader.close();
-  }
-
-  @Test
-  public void testBlurOutputFormatOverFlowMultipleReducersTest() throws IOException, InterruptedException,
-      ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(2);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 2);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setIndexLocally(job, false);
-    BlurOutputFormat.setDocumentBufferStrategy(job, DocumentBufferStrategyHeapSize.class);
-    BlurOutputFormat.setMaxDocumentBufferHeapSize(job, 128 * 1024);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    long total = 0;
-    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
-      Path path = new Path(output, ShardUtil.getShardName(i));
-      Collection<Path> commitedTasks = getCommitedTasks(path);
-      assertEquals(1, commitedTasks.size());
-
-      DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
-      total += reader.numDocs();
-      reader.close();
-    }
-    assertEquals(80000, total);
-
-  }
-
-  @Test
-  public void testBlurOutputFormatOverFlowMultipleReducersWithReduceMultiplierTest() throws IOException,
-      InterruptedException, ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(7);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 7);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    int multiple = 2;
-    BlurOutputFormat.setReducerMultiplier(job, multiple);
-
-    assertTrue(job.waitForCompletion(true));
-    Counters ctrs = job.getCounters();
-    System.out.println("Counters: " + ctrs);
-
-    long total = 0;
-    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
-      Path path = new Path(output, ShardUtil.getShardName(i));
-      Collection<Path> commitedTasks = getCommitedTasks(path);
-      assertTrue(multiple >= commitedTasks.size());
-      for (Path p : commitedTasks) {
-        DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, p));
-        total += reader.numDocs();
-        reader.close();
-      }
-    }
-    assertEquals(80000, total);
-
-  }
-
-  @Test(expected = IllegalArgumentException.class)
-  public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException,
-      ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-    writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
-    writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 1);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setReducerMultiplier(job, 2);
-    job.setNumReduceTasks(4);
-    job.submit();
-
-  }
-
-  // @TODO this test to fail sometimes due to issues in the MR MiniCluster
-  // @Test
-  public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException,
-      ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
-    writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
-    writeRecordsFile("in/part2", 1, 5000, 2000, 100, "cf1"); // 100 * 5000 =
-                                                             // 500,000
-
-    Job job = Job.getInstance(conf, "blur index");
-    job.setJarByClass(BlurOutputFormatTest.class);
-    job.setMapperClass(CsvBlurMapper.class);
-    job.setInputFormatClass(TrackingTextInputFormat.class);
-
-    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
-    CsvBlurMapper.addColumns(job, "cf1", "col");
-
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setShardCount(2);
-    tableDescriptor.setTableUri(new Path(TEST_ROOT_DIR + "/table/test").toString());
-    tableDescriptor.setName("test");
-
-    createShardDirectories(outDir, 2);
-
-    BlurOutputFormat.setupJob(job, tableDescriptor);
-    Path output = new Path(TEST_ROOT_DIR + "/out");
-    BlurOutputFormat.setOutputPath(job, output);
-    BlurOutputFormat.setIndexLocally(job, false);
-
-    job.submit();
-    boolean killCalled = false;
-    while (!job.isComplete()) {
-      Thread.sleep(1000);
-      System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100,
-          job.reduceProgress() * 100);
-      if (job.reduceProgress() > 0.7 && !killCalled) {
-        job.killJob();
-        killCalled = true;
-      }
-    }
-
-    assertFalse(job.isSuccessful());
-
-    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
-      Path path = new Path(output, ShardUtil.getShardName(i));
-      FileSystem fileSystem = path.getFileSystem(job.getConfiguration());
-      FileStatus[] listStatus = fileSystem.listStatus(path);
-      assertEquals(toString(listStatus), 0, listStatus.length);
-    }
-  }
-
-  private String toString(FileStatus[] listStatus) {
-    if (listStatus == null || listStatus.length == 0) {
-      return "";
-    }
-    String s = "";
-    for (FileStatus fileStatus : listStatus) {
-      if (s.length() > 0) {
-        s += ",";
-      }
-      s += fileStatus.getPath();
-    }
-    return s;
-  }
-
-  public static String readFile(String name) throws IOException {
-    DataInputStream f = localFs.open(new Path(TEST_ROOT_DIR + "/" + name));
-    BufferedReader b = new BufferedReader(new InputStreamReader(f));
-    StringBuilder result = new StringBuilder();
-    String line = b.readLine();
-    while (line != null) {
-      result.append(line);
-      result.append('\n');
-      line = b.readLine();
-    }
-    b.close();
-    return result.toString();
-  }
-
-  private Path writeRecordsFile(String name, int starintgRowId, int numberOfRows, int startRecordId,
-      int numberOfRecords, String family) throws IOException {
-    // "1,1,cf1,val1"
-    Path file = new Path(TEST_ROOT_DIR + "/" + name);
-    localFs.delete(file, false);
-    DataOutputStream f = localFs.create(file);
-    PrintWriter writer = new PrintWriter(f);
-    for (int row = 0; row < numberOfRows; row++) {
-      for (int record = 0; record < numberOfRecords; record++) {
-        writer.println(getRecord(row + starintgRowId, record + startRecordId, family));
-      }
-    }
-    writer.close();
-    return file;
-  }
-
-  private void createShardDirectories(Path outDir, int shardCount) throws IOException {
-    localFs.mkdirs(outDir);
-    for (int i = 0; i < shardCount; i++) {
-      localFs.mkdirs(new Path(outDir, ShardUtil.getShardName(i)));
-    }
-  }
-
-  private String getRecord(int rowId, int recordId, String family) {
-    return rowId + "," + recordId + "," + family + ",valuetoindex";
-  }
-}


Mime
View raw message