incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From twilli...@apache.org
Subject git commit: Let hdfs field manager ignore files he don't understand
Date Tue, 25 Feb 2014 23:33:35 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/apache-blur-0.2 b726909c4 -> e71d44ab7


Let hdfs field manager ignore files he don't understand


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e71d44ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e71d44ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e71d44ab

Branch: refs/heads/apache-blur-0.2
Commit: e71d44ab738120d12dc22f4efb5dee65bb7de031
Parents: b726909
Author: twilliams <twilliams@apache.org>
Authored: Tue Feb 25 18:33:15 2014 -0500
Committer: twilliams <twilliams@apache.org>
Committed: Tue Feb 25 18:33:15 2014 -0500

----------------------------------------------------------------------
 .../apache/blur/analysis/HdfsFieldManager.java  | 11 +++----
 .../blur/analysis/HdfsFieldManagerTest.java     | 31 ++++++++++++++++++--
 2 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index e2abfe3..4c311cd 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -60,6 +60,7 @@ public class HdfsFieldManager extends BaseFieldManager {
   private static final String FAMILY = "_family_";
   private static final String COLUMN_NAME = "_columnName_";
   private static final String SUB_COLUMN_NAME = "_subColumnName_";
+  private static final String TYPE_FILE_EXT = ".type";
 
   private static Lock _lock = new Lock() {
     private final java.util.concurrent.locks.Lock _javalock = new ReentrantReadWriteLock().writeLock();
@@ -104,10 +105,10 @@ public class HdfsFieldManager extends BaseFieldManager {
       FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new PathFilter() {
         @Override
         public boolean accept(Path path) {
-          if (path.getName().endsWith(".tmp")) {
-            return false;
+          if (path.getName().endsWith(TYPE_FILE_EXT)) {
+            return true;
           }
-          return true;
+          return false;
         }
       });
       if (listStatus == null) {
@@ -116,7 +117,7 @@ public class HdfsFieldManager extends BaseFieldManager {
       List<String> fieldNames = new ArrayList<String>();
       for (FileStatus fileStatus : listStatus) {
         if (!fileStatus.isDir()) {
-          fieldNames.add(fileStatus.getPath().getName());
+          fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT, ""));
         }
       }
       return fieldNames;
@@ -188,7 +189,7 @@ public class HdfsFieldManager extends BaseFieldManager {
   }
 
   private Path getFieldPath(String fieldName) {
-    return new Path(_storagePath, fieldName);
+    return new Path(_storagePath, fieldName + TYPE_FILE_EXT);
   }
 
   private String getComments() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
index 26169a5..f14fb8c 100644
--- a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
+++ b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
@@ -1,13 +1,18 @@
 package org.apache.blur.analysis;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
@@ -30,17 +35,37 @@ import org.junit.Test;
  * the License.
  */
 public class HdfsFieldManagerTest extends BaseFieldManagerTest {
+  private static final String DFS_FIELD_MANAGER_PATH = "./target/tmp/HdfsFieldManagerTest/meta";
+  
   @Override
   protected BaseFieldManager newFieldManager(boolean create) throws IOException {
     Configuration config = new Configuration();
-    Path path = new Path("./target/tmp/HdfsFieldManagerTest/meta");
+    Path path = new Path(DFS_FIELD_MANAGER_PATH);
     FileSystem fileSystem = path.getFileSystem(config);
     if (create) {
       fileSystem.delete(path, true);
     }
     return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config);
   }
-
+  
+    @Test
+    public void fieldManagerShouldIgnoreUnknownFiles() throws IOException {
+      BaseFieldManager fieldManager = newFieldManager(true);
+      fieldManager.addColumnDefinition("fam1", "col1", null, true, "text", false, null);
+      FieldTypeDefinition fieldTypeDefinition1 = fieldManager.getFieldTypeDefinition("fam1.col1");
+      
+      assertNotNull(fieldTypeDefinition1);
+      Path path = new Path(DFS_FIELD_MANAGER_PATH, "mydoc.txt");
+      FSDataOutputStream outputStream = path.getFileSystem(new Configuration()).create(path,
false);
+      IOUtils.write("Some text..", outputStream);
+      outputStream.close();
+      
+      fieldManager.loadFromStorage();
+      
+      FieldTypeDefinition fieldTypeDefinition2 = fieldManager.getFieldTypeDefinition("fam1.col1");
+      assertNotNull(fieldTypeDefinition2);
+  }
+    
   @SuppressWarnings("unchecked")
   @Test
   public void testStoreMetaData() throws IOException {


Mime
View raw message