incubator-blur-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Aaron McCurry <amccu...@gmail.com>
Subject Re: git commit: Let hdfs field manager ignore files he don't understand
Date Wed, 26 Feb 2014 00:49:48 GMT
@@ -116,7 +117,7 @@ public class HdfsFieldManager extends BaseFieldManager {
       List<String> fieldNames = new ArrayList<String>();
       for (FileStatus fileStatus : listStatus) {
         if (!fileStatus.isDir()) {
-          fieldNames.add(fileStatus.getPath().getName());
+          fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT,
""));
         }
       }

Do you think that this will cause problems for fields that are <some
family>.type<any suffix> ?  Maybe we should check that the file ends with
the TYPE_FILE_EXT and substring up to the beginning of the suffix?

Aaron



On Tue, Feb 25, 2014 at 6:33 PM, <twilliams@apache.org> wrote:

> Repository: incubator-blur
> Updated Branches:
>   refs/heads/apache-blur-0.2 b726909c4 -> e71d44ab7
>
>
> Let hdfs field manager ignore files he don't understand
>
>
> Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
> Commit:
> http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e71d44ab
> Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e71d44ab
> Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e71d44ab
>
> Branch: refs/heads/apache-blur-0.2
> Commit: e71d44ab738120d12dc22f4efb5dee65bb7de031
> Parents: b726909
> Author: twilliams <twilliams@apache.org>
> Authored: Tue Feb 25 18:33:15 2014 -0500
> Committer: twilliams <twilliams@apache.org>
> Committed: Tue Feb 25 18:33:15 2014 -0500
>
> ----------------------------------------------------------------------
>  .../apache/blur/analysis/HdfsFieldManager.java  | 11 +++----
>  .../blur/analysis/HdfsFieldManagerTest.java     | 31 ++++++++++++++++++--
>  2 files changed, 34 insertions(+), 8 deletions(-)
> ----------------------------------------------------------------------
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> ----------------------------------------------------------------------
> diff --git
> a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> index e2abfe3..4c311cd 100644
> ---
> a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> +++
> b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> @@ -60,6 +60,7 @@ public class HdfsFieldManager extends BaseFieldManager {
>    private static final String FAMILY = "_family_";
>    private static final String COLUMN_NAME = "_columnName_";
>    private static final String SUB_COLUMN_NAME = "_subColumnName_";
> +  private static final String TYPE_FILE_EXT = ".type";
>
>    private static Lock _lock = new Lock() {
>      private final java.util.concurrent.locks.Lock _javalock = new
> ReentrantReadWriteLock().writeLock();
> @@ -104,10 +105,10 @@ public class HdfsFieldManager extends
> BaseFieldManager {
>        FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new
> PathFilter() {
>          @Override
>          public boolean accept(Path path) {
> -          if (path.getName().endsWith(".tmp")) {
> -            return false;
> +          if (path.getName().endsWith(TYPE_FILE_EXT)) {
> +            return true;
>            }
> -          return true;
> +          return false;
>          }
>        });
>        if (listStatus == null) {
> @@ -116,7 +117,7 @@ public class HdfsFieldManager extends BaseFieldManager
> {
>        List<String> fieldNames = new ArrayList<String>();
>        for (FileStatus fileStatus : listStatus) {
>          if (!fileStatus.isDir()) {
> -          fieldNames.add(fileStatus.getPath().getName());
> +
>  fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT, ""));
>          }
>        }
>        return fieldNames;
> @@ -188,7 +189,7 @@ public class HdfsFieldManager extends BaseFieldManager
> {
>    }
>
>    private Path getFieldPath(String fieldName) {
> -    return new Path(_storagePath, fieldName);
> +    return new Path(_storagePath, fieldName + TYPE_FILE_EXT);
>    }
>
>    private String getComments() {
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> ----------------------------------------------------------------------
> diff --git
> a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> index 26169a5..f14fb8c 100644
> ---
> a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> +++
> b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> @@ -1,13 +1,18 @@
>  package org.apache.blur.analysis;
>
> -import static org.junit.Assert.*;
> +import static org.junit.Assert.assertFalse;
> +import static org.junit.Assert.assertNotNull;
> +import static org.junit.Assert.assertNull;
> +import static org.junit.Assert.assertTrue;
>
>  import java.io.IOException;
>  import java.util.HashMap;
>  import java.util.Map;
>  import java.util.Map.Entry;
>
> +import org.apache.commons.io.IOUtils;
>  import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.fs.FSDataOutputStream;
>  import org.apache.hadoop.fs.FileSystem;
>  import org.apache.hadoop.fs.Path;
>  import org.apache.lucene.analysis.core.KeywordAnalyzer;
> @@ -30,17 +35,37 @@ import org.junit.Test;
>   * the License.
>   */
>  public class HdfsFieldManagerTest extends BaseFieldManagerTest {
> +  private static final String DFS_FIELD_MANAGER_PATH =
> "./target/tmp/HdfsFieldManagerTest/meta";
> +
>    @Override
>    protected BaseFieldManager newFieldManager(boolean create) throws
> IOException {
>      Configuration config = new Configuration();
> -    Path path = new Path("./target/tmp/HdfsFieldManagerTest/meta");
> +    Path path = new Path(DFS_FIELD_MANAGER_PATH);
>      FileSystem fileSystem = path.getFileSystem(config);
>      if (create) {
>        fileSystem.delete(path, true);
>      }
>      return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(),
> path, config);
>    }
> -
> +
> +    @Test
> +    public void fieldManagerShouldIgnoreUnknownFiles() throws IOException
> {
> +      BaseFieldManager fieldManager = newFieldManager(true);
> +      fieldManager.addColumnDefinition("fam1", "col1", null, true,
> "text", false, null);
> +      FieldTypeDefinition fieldTypeDefinition1 =
> fieldManager.getFieldTypeDefinition("fam1.col1");
> +
> +      assertNotNull(fieldTypeDefinition1);
> +      Path path = new Path(DFS_FIELD_MANAGER_PATH, "mydoc.txt");
> +      FSDataOutputStream outputStream = path.getFileSystem(new
> Configuration()).create(path, false);
> +      IOUtils.write("Some text..", outputStream);
> +      outputStream.close();
> +
> +      fieldManager.loadFromStorage();
> +
> +      FieldTypeDefinition fieldTypeDefinition2 =
> fieldManager.getFieldTypeDefinition("fam1.col1");
> +      assertNotNull(fieldTypeDefinition2);
> +  }
> +
>    @SuppressWarnings("unchecked")
>    @Test
>    public void testStoreMetaData() throws IOException {
>
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message